diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..44621208ff7748ac8548ea8ef4d0fae080c802e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 2048 +capacity: 1.00 +expert_dispatch_overlap: off +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6000 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 230 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.json new file mode 100644 index 0000000000000000000000000000000000000000..29eed6f3f24da72f39e06e9da9903d389f4e2ca1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 31640019968, + "active_bytes_peak": 79683648512, + "active_bytes_peak_delta": 44217312256, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79683648512, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79683648512, + "device": 0, + "inactive_split_bytes_current": 4857414656, + "inactive_split_bytes_peak": 5749855232, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 94378131456, + "reserved_bytes_peak": 94378131456, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T11:40:12.483106Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 31640019968, + "allocated_bytes_peak": 79683648512, + "allocated_bytes_peak_delta": 44217312256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T11:39:04.476196Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4857414656, + "inactive_split_bytes_current_delta": 4722498560, + "inactive_split_bytes_peak": 5749855232, + "inactive_split_bytes_peak_delta": 5589223424, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 94378131456, + "reserved_bytes_current_delta": 55132028928, + "reserved_bytes_peak": 94378131456, + "reserved_bytes_peak_delta": 55132028928, + "segment_count_current": 518, + "segment_count_current_delta": 446, + "segment_count_peak": 518, + "segment_count_peak_delta": 446, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.pickle", + "timestamp_utc": "2026-08-01T11:40:12.592236Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14203ccb7eb8de722d0dedca871d9dbdafcf8b90 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114012483613Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13eb403a964d4aaa0e195ef80d1de987cbc685f88b0a65095ec8c26eff635061 +size 5383895 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb9230132d781ecff95fac459c0823cff3d08ed3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322287104, + "active_bytes_peak_delta": 23638638592, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322287104, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322287104, + "device": 0, + "inactive_split_bytes_current": 3961930752, + "inactive_split_bytes_peak": 5749855232, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104406712320, + "reserved_bytes_peak": 104406712320, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T11:40:13.440234Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322287104, + "allocated_bytes_peak_delta": 23638638592, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79683648512, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79683648512, + "device": 0, + "inactive_split_bytes_current": 4857414656, + "inactive_split_bytes_peak": 5749855232, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 94378131456, + "reserved_bytes_peak": 94378131456, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T11:40:12.611327Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3961930752, + "inactive_split_bytes_current_delta": -895483904, + "inactive_split_bytes_peak": 5749855232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104406712320, + "reserved_bytes_current_delta": 10028580864, + "reserved_bytes_peak": 104406712320, + "reserved_bytes_peak_delta": 10028580864, + "segment_count_current": 591, + "segment_count_current_delta": 73, + "segment_count_peak": 591, + "segment_count_peak_delta": 73, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.pickle", + "timestamp_utc": "2026-08-01T11:40:13.538384Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..850884ed5bf8c51613be20b0cedd9f2f6aaa6563 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114013440639Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7770105a42e873f83f3788b3924e707562035434d844fe87fa2cfc5ae405aaee +size 5583228 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e4e565caad364eff6297721fd9266eee35c270d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322287104, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322287104, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322287104, + "device": 0, + "inactive_split_bytes_current": 4131800064, + "inactive_split_bytes_peak": 5749855232, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104406712320, + "reserved_bytes_peak": 104406712320, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T11:40:14.352612Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322287104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322287104, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322287104, + "device": 0, + "inactive_split_bytes_current": 3961930752, + "inactive_split_bytes_peak": 5749855232, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104406712320, + "reserved_bytes_peak": 104406712320, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T11:40:13.549825Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4131800064, + "inactive_split_bytes_current_delta": 169869312, + "inactive_split_bytes_peak": 5749855232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104406712320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104406712320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 591, + "segment_count_current_delta": 0, + "segment_count_peak": 591, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.pickle", + "timestamp_utc": "2026-08-01T11:40:14.444958Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..262ba074c449dfeeac7375eea289e27cac895d70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114014352990Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:900b45d88f1d794f2575cd36e05bc551a33c2bd3d82fbc54ed027bbfdf34e584 +size 5584389 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fca7956a99c772a89588d73b255572992fa20d4b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121491349504, + "active_bytes_peak_delta": 18169062400, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_peak": 7220541440, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_peak": 132521132032, + "segment_count_current": 798, + "segment_count_peak": 798, + "timestamp_utc": "2026-08-01T11:40:17.033382Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103336542208, + "allocated_bytes_peak_delta": 14255104, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322287104, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322287104, + "device": 0, + "inactive_split_bytes_current": 4131800064, + "inactive_split_bytes_peak": 5749855232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104406712320, + "reserved_bytes_peak": 104406712320, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T11:40:14.455675Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_current_delta": 452718592, + "inactive_split_bytes_peak": 7220541440, + "inactive_split_bytes_peak_delta": 1470686208, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_current_delta": 28114419712, + "reserved_bytes_peak": 132521132032, + "reserved_bytes_peak_delta": 28114419712, + "segment_count_current": 798, + "segment_count_current_delta": 207, + "segment_count_peak": 798, + "segment_count_peak_delta": 207, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.150803Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..086ecacd50225dab125953602dd051a2b17aac19 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114017033987Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3353b418b19312db946e05d6a17b3e9d5b702632e9c28904515cff97fe2cce6 +size 6274771 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1a5abf13e40d2f8627ea4ed36a9e4cd187d11c46 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121491349504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_peak": 7220541440, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_peak": 132521132032, + "segment_count_current": 798, + "segment_count_peak": 798, + "timestamp_utc": "2026-08-01T11:40:17.864648Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103336542208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_peak": 7220541440, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_peak": 132521132032, + "segment_count_current": 798, + "segment_count_peak": 798, + "timestamp_utc": "2026-08-01T11:40:17.164045Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7220541440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132521132032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 798, + "segment_count_current_delta": 0, + "segment_count_peak": 798, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.872215Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d100f274c2cd9790e09ff8b3c3c1e298173e7d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114017865085Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6435a311776ea8d99a13db28debafd039dbda3f4b899a139c6a412d0070e9fa7 +size 266048 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6b38782c79a74b9227d48995a0fa76f36cfd07ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121491349504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_peak": 7220541440, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_peak": 132521132032, + "segment_count_current": 798, + "segment_count_peak": 798, + "timestamp_utc": "2026-08-01T11:40:18.565796Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103336542208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_peak": 7220541440, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_peak": 132521132032, + "segment_count_current": 798, + "segment_count_peak": 798, + "timestamp_utc": "2026-08-01T11:40:17.873174Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7220541440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132521132032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 798, + "segment_count_current_delta": 0, + "segment_count_peak": 798, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.pickle", + "timestamp_utc": "2026-08-01T11:40:18.572888Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60a73f00b8ac5b7a0f0156b9119aa939d67d0b81 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114018566187Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e12dcf81fed3acc951c00dde72513e207be4b99b8f85eac0f1e26ccc122d9c8 +size 266049 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.json new file mode 100644 index 0000000000000000000000000000000000000000..586071480d0f23e6b2bb4982d86aa281a8c0468d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121491349504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_peak": 7220541440, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_peak": 132521132032, + "segment_count_current": 798, + "segment_count_peak": 798, + "timestamp_utc": "2026-08-01T11:40:19.267583Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103336542208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_peak": 7220541440, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_peak": 132521132032, + "segment_count_current": 798, + "segment_count_peak": 798, + "timestamp_utc": "2026-08-01T11:40:18.573818Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7220541440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132521132032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 798, + "segment_count_current_delta": 0, + "segment_count_peak": 798, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.275794Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60fdc569d55d424aff6df9259256a125a4d7fe07 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114019268050Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d3325263acc1c15b76e149c7e8d846bdff5c56966c03cbdc5967e9edf662aaf +size 266049 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c09ad4312ba6da9c8ecda9be2795fa97c3c8da0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121491349504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_peak": 7220541440, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_peak": 132521132032, + "segment_count_current": 798, + "segment_count_peak": 798, + "timestamp_utc": "2026-08-01T11:40:19.968607Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103336542208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_peak": 7220541440, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_peak": 132521132032, + "segment_count_current": 798, + "segment_count_peak": 798, + "timestamp_utc": "2026-08-01T11:40:19.276766Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7220541440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132521132032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 798, + "segment_count_current_delta": 0, + "segment_count_peak": 798, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.975851Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..78188950833f69eb29044c3a84fcfc14ee85cb36 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114019969019Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:058c23d59cf99ce1d34d3e898fccaa8bd00d8cea459c66f29cbde75e279054b4 +size 266049 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31e123123f170b0429783c1e8c8e227dc8dd3458 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121491349504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_peak": 7220541440, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_peak": 132521132032, + "segment_count_current": 798, + "segment_count_peak": 798, + "timestamp_utc": "2026-08-01T11:40:20.672461Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103336542208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_peak": 7220541440, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_peak": 132521132032, + "segment_count_current": 798, + "segment_count_peak": 798, + "timestamp_utc": "2026-08-01T11:40:19.976811Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7220541440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132521132032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 798, + "segment_count_current_delta": 0, + "segment_count_peak": 798, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.pickle", + "timestamp_utc": "2026-08-01T11:40:20.679449Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cd3250cf1572f10f7a998c7e39b162d7a535ed1a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114020672838Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:497816d6d6e28441383edc174a3ef28054bf0ac55df61c8863520df789c7791f +size 266049 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2dca838e32ac26df9c6296458758caecf1744f82 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854884352, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 121491349504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854884352, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66854884352, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 2523098112, + "inactive_split_bytes_peak": 7220541440, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132544200704, + "reserved_bytes_peak": 132544200704, + "segment_count_current": 800, + "segment_count_peak": 800, + "timestamp_utc": "2026-08-01T11:41:13.363803Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854884352, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103336542208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121491349504, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103336542208, + "device": 0, + "inactive_split_bytes_current": 4584518656, + "inactive_split_bytes_peak": 7220541440, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 132521132032, + "reserved_bytes_peak": 132521132032, + "segment_count_current": 798, + "segment_count_peak": 798, + "timestamp_utc": "2026-08-01T11:40:20.680382Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2523098112, + "inactive_split_bytes_current_delta": -2061420544, + "inactive_split_bytes_peak": 7220541440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132544200704, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132544200704, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 800, + "segment_count_current_delta": 2, + "segment_count_peak": 800, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.pickle", + "timestamp_utc": "2026-08-01T11:41:13.448813Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..70ae9fb16167aa2b856ac7c90561e2f8236d3656 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114113364279Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691713541afb401c4594240700d7746937afee4116911dc1465e490b845e29e1 +size 4204922 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36a5afe933fb9547a705b5a8d029fc0be7a5296a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 31640019968, + "active_bytes_peak": 79690286080, + "active_bytes_peak_delta": 44223949824, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79690286080, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79690286080, + "device": 1, + "inactive_split_bytes_current": 4328932352, + "inactive_split_bytes_peak": 5229261824, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 94376034304, + "reserved_bytes_peak": 94376034304, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T11:40:12.483218Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 31640019968, + "allocated_bytes_peak": 79690286080, + "allocated_bytes_peak_delta": 44223949824, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:39:04.476570Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4328932352, + "inactive_split_bytes_current_delta": 4196113408, + "inactive_split_bytes_peak": 5229261824, + "inactive_split_bytes_peak_delta": 5068875776, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 94376034304, + "reserved_bytes_current_delta": 55132028928, + "reserved_bytes_peak": 94376034304, + "reserved_bytes_peak_delta": 55132028928, + "segment_count_current": 520, + "segment_count_current_delta": 449, + "segment_count_peak": 520, + "segment_count_peak_delta": 449, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.pickle", + "timestamp_utc": "2026-08-01T11:40:12.600020Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b0e46d7b5727e2626288d9994aa077a8cb7ea50d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114012483826Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81b303ab50c373505e17e5a08d61daf215d49515fb8d51512f5579ea9dc08012 +size 5379054 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a8cd4228fd64eee93eb9fb605e9d980d467d4cec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103326691328, + "active_bytes_peak_delta": 23636405248, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103326691328, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 3620094976, + "inactive_split_bytes_peak": 5229261824, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T11:40:13.440274Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691328, + "allocated_bytes_peak_delta": 23636405248, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79690286080, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79690286080, + "device": 1, + "inactive_split_bytes_current": 4328932352, + "inactive_split_bytes_peak": 5229261824, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 94376034304, + "reserved_bytes_peak": 94376034304, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T11:40:12.617749Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3620094976, + "inactive_split_bytes_current_delta": -708837376, + "inactive_split_bytes_peak": 5229261824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104421392384, + "reserved_bytes_current_delta": 10045358080, + "reserved_bytes_peak": 104421392384, + "reserved_bytes_peak_delta": 10045358080, + "segment_count_current": 590, + "segment_count_current_delta": 70, + "segment_count_peak": 590, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.pickle", + "timestamp_utc": "2026-08-01T11:40:13.545141Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4034b33b1531fbaee686fe4c96239c4740a93e0c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114013440672Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c9035cf43d8fc94c39de8f7a9fc59b0c2de1afe3ff5e2e3ec9392788bc29574 +size 5578891 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ee260b2e5f6b29b8d2a9b6c01c09e5e20921d40 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103326691328, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103326691328, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 3712369664, + "inactive_split_bytes_peak": 5229261824, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T11:40:14.352684Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103326691328, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 3620094976, + "inactive_split_bytes_peak": 5229261824, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T11:40:13.558680Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3712369664, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 5229261824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104421392384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104421392384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.pickle", + "timestamp_utc": "2026-08-01T11:40:14.456557Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..56049d90d703e7d563ace37ca73fa6a04d5037e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114014353089Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:919b3d739ef62345a6435a45506cc04af73b5428803af66af59c963371607c48 +size 5581958 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b8995a21ea4c350122e10edc5a5f0558a733cdcd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 18150764544, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_peak": 7682149376, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132101701632, + "reserved_bytes_peak": 132101701632, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:17.033418Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103326691328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103326691328, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 3712369664, + "inactive_split_bytes_peak": 5229261824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T11:40:14.469703Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_current_delta": 536604672, + "inactive_split_bytes_peak": 7682149376, + "inactive_split_bytes_peak_delta": 2452887552, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132101701632, + "reserved_bytes_current_delta": 27680309248, + "reserved_bytes_peak": 132101701632, + "reserved_bytes_peak_delta": 27680309248, + "segment_count_current": 790, + "segment_count_current_delta": 200, + "segment_count_peak": 790, + "segment_count_peak_delta": 200, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.153033Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aa5f034d1a01c6e01b41219a2c50f74d2b5925c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114017033984Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26ccef36f0e9d2cec4a938fee739eb69e8ff1132ac1575aab7473826237fbc75 +size 5852185 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5a0680880ae8cf570a98f9c50db965a1ed783b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_peak": 7682149376, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132101701632, + "reserved_bytes_peak": 132101701632, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:17.864704Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_peak": 7682149376, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 132101701632, + "reserved_bytes_peak": 132101701632, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:17.168178Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7682149376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132101701632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132101701632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 790, + "segment_count_current_delta": 0, + "segment_count_peak": 790, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.872594Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..36d3e7bb982bd62ca7666d9b98913e3ce1dae6a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114017865186Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:621a7e04293676d762c0de3bc2668b3b5ae6f5ebe02fb676dcf05f92e211d2dc +size 243291 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80f099f1f9416b8b9ae6ff5d973cdcb7bbfa0428 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_peak": 7682149376, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132101701632, + "reserved_bytes_peak": 132101701632, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:18.566062Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_peak": 7682149376, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 132101701632, + "reserved_bytes_peak": 132101701632, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:17.873601Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7682149376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132101701632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132101701632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 790, + "segment_count_current_delta": 0, + "segment_count_peak": 790, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.pickle", + "timestamp_utc": "2026-08-01T11:40:18.574762Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8e6b1bb8184b7f5a927ce8ea171533e7dd56fabd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114018566475Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:203517b91270fdd338a0320f887d94b09fe296284c61cb7a3d3d215f631adea6 +size 243291 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d93b9b737fe433cedd0f5e150b00da39fc541c7a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_peak": 7682149376, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132101701632, + "reserved_bytes_peak": 132101701632, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:19.267574Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_peak": 7682149376, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 132101701632, + "reserved_bytes_peak": 132101701632, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:18.575728Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7682149376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132101701632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132101701632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 790, + "segment_count_current_delta": 0, + "segment_count_peak": 790, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.274628Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5bf657c5c6ce0d8102b9560a1be120b88ba8a926 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114019267965Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84daf2896871a2726a78c16d3263a77aa76eb557618b149c658242774e14beba +size 243291 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a71ceb703b39b2155119cfb54206dd9cbc6adf4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_peak": 7682149376, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132101701632, + "reserved_bytes_peak": 132101701632, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:19.968643Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_peak": 7682149376, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 132101701632, + "reserved_bytes_peak": 132101701632, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:19.275567Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7682149376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132101701632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132101701632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 790, + "segment_count_current_delta": 0, + "segment_count_peak": 790, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.975826Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c54ae0521e7249a68750150d946cfeedece82f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114019969053Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:491e864bb6d6cebfc7d4d0186fc45e79cd23addbdc1793bbe2e6978bf57b54c5 +size 243291 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dc575b63f9f37714f8d128af12f0a32b343f8edb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_peak": 7682149376, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132101701632, + "reserved_bytes_peak": 132101701632, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:20.672505Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_peak": 7682149376, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 132101701632, + "reserved_bytes_peak": 132101701632, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:19.976764Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7682149376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132101701632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132101701632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 790, + "segment_count_current_delta": 0, + "segment_count_peak": 790, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.pickle", + "timestamp_utc": "2026-08-01T11:40:20.679586Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6d3b9a235b33f7c5ec05439149db675534ddee87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114020672901Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eafba8982e23688b1a834c2913971f061d8f8840902cd0bbc65619f83b29868c +size 243291 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65d62b0ecbc7e81fc943899bc397a02fa24c5e45 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854884352, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854884352, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66854884352, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 2296605696, + "inactive_split_bytes_peak": 7682149376, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 792, + "segment_count_peak": 792, + "timestamp_utc": "2026-08-01T11:41:13.363810Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854884352, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103326691328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103326691328, + "device": 1, + "inactive_split_bytes_current": 4248974336, + "inactive_split_bytes_peak": 7682149376, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 132101701632, + "reserved_bytes_peak": 132101701632, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:20.680576Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2296605696, + "inactive_split_bytes_current_delta": -1952368640, + "inactive_split_bytes_peak": 7682149376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132124770304, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132124770304, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 792, + "segment_count_current_delta": 2, + "segment_count_peak": 792, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.pickle", + "timestamp_utc": "2026-08-01T11:41:13.451555Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a9e0e262ac6647b0e4b196c9b710a89045daba5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114113364282Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2e99df3ac0599cd00e64a13265bc0d4fe58af3467079b99f62d72165aa9eab7 +size 4121826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04868f558a5fb5ec726e13e236b26cf77b969079 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 31640019968, + "active_bytes_peak": 79690286080, + "active_bytes_peak_delta": 44223949824, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79690286080, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79690286080, + "device": 2, + "inactive_split_bytes_current": 4328932352, + "inactive_split_bytes_peak": 5464060928, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93906272256, + "reserved_bytes_peak": 93906272256, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T11:40:12.483142Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 31640019968, + "allocated_bytes_peak": 79690286080, + "allocated_bytes_peak_delta": 44223949824, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:39:04.476596Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4328932352, + "inactive_split_bytes_current_delta": 4196113408, + "inactive_split_bytes_peak": 5464060928, + "inactive_split_bytes_peak_delta": 5303674880, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93906272256, + "reserved_bytes_current_delta": 54662266880, + "reserved_bytes_peak": 93906272256, + "reserved_bytes_peak_delta": 54662266880, + "segment_count_current": 517, + "segment_count_current_delta": 446, + "segment_count_peak": 517, + "segment_count_peak_delta": 446, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.pickle", + "timestamp_utc": "2026-08-01T11:40:12.593828Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..39e2150be49047c9bcb990b419d9cde7e70ecad5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114012483762Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba85767f668dba66ab14017ea6b62d73b66eea84f68ff3cc013c9ba27273d4d +size 5378522 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.json new file mode 100644 index 0000000000000000000000000000000000000000..64dfe28f34b0833059db767768b3f11718920726 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103326691328, + "active_bytes_peak_delta": 23636405248, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103326691328, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103326691328, + "device": 2, + "inactive_split_bytes_current": 4033233920, + "inactive_split_bytes_peak": 5464060928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 589, + "segment_count_peak": 589, + "timestamp_utc": "2026-08-01T11:40:13.440247Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691328, + "allocated_bytes_peak_delta": 23636405248, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79690286080, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79690286080, + "device": 2, + "inactive_split_bytes_current": 4328932352, + "inactive_split_bytes_peak": 5464060928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 93906272256, + "reserved_bytes_peak": 93906272256, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T11:40:12.611672Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4033233920, + "inactive_split_bytes_current_delta": -295698432, + "inactive_split_bytes_peak": 5464060928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104421392384, + "reserved_bytes_current_delta": 10515120128, + "reserved_bytes_peak": 104421392384, + "reserved_bytes_peak_delta": 10515120128, + "segment_count_current": 589, + "segment_count_current_delta": 72, + "segment_count_peak": 589, + "segment_count_peak_delta": 72, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.pickle", + "timestamp_utc": "2026-08-01T11:40:13.545258Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3888da2a6e7739ca1f4f5412a052c1a50aa21d45 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114013440660Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2c95ab6ab4b590631e5c6111813d6e9e421b61faaca37e1d100e8d2b6ed24d9 +size 5574765 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e631e9b9ecd54cabb7c241fa92248e2ec2cffb0f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103329097728, + "active_bytes_peak_delta": 2406400, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103329097728, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 3735438336, + "inactive_split_bytes_peak": 5464060928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 589, + "segment_count_peak": 589, + "timestamp_utc": "2026-08-01T11:40:14.352653Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 2406400, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103326691328, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103326691328, + "device": 2, + "inactive_split_bytes_current": 4033233920, + "inactive_split_bytes_peak": 5464060928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 589, + "segment_count_peak": 589, + "timestamp_utc": "2026-08-01T11:40:13.558605Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3735438336, + "inactive_split_bytes_current_delta": -297795584, + "inactive_split_bytes_peak": 5464060928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104421392384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104421392384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 589, + "segment_count_current_delta": 0, + "segment_count_peak": 589, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.pickle", + "timestamp_utc": "2026-08-01T11:40:14.453082Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dcf0775dd51cd5ab813afa94142073559f66b560 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114014353057Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84cb620613bf3d7a35b8afc61b5ec87a3df4a82cd8a767258abbd7428bca44df +size 5578752 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2596f943ac67aa8d59204cb3604b189f4c444bfb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 18148358144, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_peak": 7711509504, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:17.033393Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103329097728, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 3735438336, + "inactive_split_bytes_peak": 5464060928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 589, + "segment_count_peak": 589, + "timestamp_utc": "2026-08-01T11:40:14.465898Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_current_delta": 389804032, + "inactive_split_bytes_peak": 7711509504, + "inactive_split_bytes_peak_delta": 2247448576, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132124770304, + "reserved_bytes_current_delta": 27703377920, + "reserved_bytes_peak": 132124770304, + "reserved_bytes_peak_delta": 27703377920, + "segment_count_current": 788, + "segment_count_current_delta": 199, + "segment_count_peak": 788, + "segment_count_peak_delta": 199, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.153026Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c90bd89a226c167deba45ca150d4681b52583d83 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114017033950Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57d6fe28e9cba49c9df5310e3ed2d64bf48bf43c5ae103a180cadecbce7c445c +size 5851567 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T114017865123Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T114017865123Z.json new file mode 100644 index 0000000000000000000000000000000000000000..154e73dd87e904c56a13c9101568341128ae198b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T114017865123Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_peak": 7711509504, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:17.864666Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_peak": 7711509504, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:17.167931Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7711509504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132124770304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132124770304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T114017865123Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.872206Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T114017865123Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T114017865123Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..79f659ded806bae0798701c656223f6cf9d2d254 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T114017865123Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ac707c304abde3c593d10e6aee85c675e6ab503b87a2f775a57adeb4c0bab82 +size 242732 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T114018566172Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T114018566172Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6e62978ff46cb98815ec603bc8cdb162d256f5c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T114018566172Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_peak": 7711509504, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:18.565774Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_peak": 7711509504, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:17.873206Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7711509504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132124770304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132124770304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T114018566172Z.pickle", + "timestamp_utc": "2026-08-01T11:40:18.572740Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T114018566172Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T114018566172Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b86bcb2abe4e59500f22186f3f06606b5f36356d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T114018566172Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b751160c2038847a5872fbdd31456a06782c7640a5d5b3fb8b49ad918c5908ec +size 242732 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T114019267912Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T114019267912Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69a2889b7b15f89b1aab3aa29cfc1278b717fb39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T114019267912Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_peak": 7711509504, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:19.267524Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_peak": 7711509504, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:18.573682Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7711509504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132124770304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132124770304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T114019267912Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.274204Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T114019267912Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T114019267912Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3843f43453d90a8825190c59805cb8859e5ff56c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T114019267912Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3d80ff91cdf66cd4e02233ff69c58d3f81368e00e38b4c549ddecb27c88a861 +size 242732 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T114019968969Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T114019968969Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ba16cf2f3c23b5d235536f445f77b77ae76049b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T114019968969Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_peak": 7711509504, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:19.968583Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_peak": 7711509504, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:19.275136Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7711509504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132124770304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132124770304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T114019968969Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.975272Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T114019968969Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T114019968969Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5c8b991293bd54379c8d91993e3abf4fff041ddf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T114019968969Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:051f14979246e5d9439d36ba1116ec12d0c70a06c2e06ac84b578e06b407eddc +size 242732 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T114020672853Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T114020672853Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3992e0bc940b9acab35707a232f10cea160c9b36 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T114020672853Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_peak": 7711509504, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:20.672474Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_peak": 7711509504, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:19.976224Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7711509504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132124770304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132124770304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T114020672853Z.pickle", + "timestamp_utc": "2026-08-01T11:40:20.679152Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T114020672853Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T114020672853Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ad284305a8744f06bb735dad09caea5b08c6512 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T114020672853Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:436cec9be5c7da2d201cab3db8e8d271adffb51ca56e2ad8ad163071a8e3e0f1 +size 242732 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T114113364298Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T114113364298Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7610ed7344c8789fcb4f0af7e008e86092d309fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T114113364298Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854884352, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854884352, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66854884352, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 2294508544, + "inactive_split_bytes_peak": 7711509504, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132147838976, + "reserved_bytes_peak": 132147838976, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:41:13.363819Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854884352, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 2, + "inactive_split_bytes_current": 4125242368, + "inactive_split_bytes_peak": 7711509504, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 132124770304, + "reserved_bytes_peak": 132124770304, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:20.680097Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2294508544, + "inactive_split_bytes_current_delta": -1830733824, + "inactive_split_bytes_peak": 7711509504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132147838976, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132147838976, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 790, + "segment_count_current_delta": 2, + "segment_count_peak": 790, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T114113364298Z.pickle", + "timestamp_utc": "2026-08-01T11:41:13.451399Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T114113364298Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T114113364298Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..087a40aec5481bcbe135971f81fa037e54f32556 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T114113364298Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4455942a6aaa3f249b7fa9af617e2ac5b7ca72965c9611f1b5ee7ad6b32b0ab +size 4079029 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T114012483800Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T114012483800Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b985916a7c0534c236ae1937a75777fcf614e271 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T114012483800Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 31640019968, + "active_bytes_peak": 79690286080, + "active_bytes_peak_delta": 44223949824, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79690286080, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79690286080, + "device": 3, + "inactive_split_bytes_current": 4094051328, + "inactive_split_bytes_peak": 5989206016, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93436510208, + "reserved_bytes_peak": 93436510208, + "segment_count_current": 510, + "segment_count_peak": 510, + "timestamp_utc": "2026-08-01T11:40:12.483203Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 31640019968, + "allocated_bytes_peak": 79690286080, + "allocated_bytes_peak_delta": 44223949824, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:39:04.476647Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4094051328, + "inactive_split_bytes_current_delta": 3961232384, + "inactive_split_bytes_peak": 5989206016, + "inactive_split_bytes_peak_delta": 5828819968, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93436510208, + "reserved_bytes_current_delta": 54192504832, + "reserved_bytes_peak": 93436510208, + "reserved_bytes_peak_delta": 54192504832, + "segment_count_current": 510, + "segment_count_current_delta": 439, + "segment_count_peak": 510, + "segment_count_peak_delta": 439, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T114012483800Z.pickle", + "timestamp_utc": "2026-08-01T11:40:12.592727Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T114012483800Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T114012483800Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1aa86e7d1052cb9aa99297629d1ec0d908b4a7f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T114012483800Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1d399f2c52139330f7d4fcd11bf4b9cc5e155d6129f6b0d34ff2932fafaf6f6 +size 5376126 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T114013440728Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T114013440728Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e985a3f6363cd90a46ce3a9fafb4634104abc9e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T114013440728Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103328264192, + "active_bytes_peak_delta": 23637978112, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103328264192, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103328264192, + "device": 3, + "inactive_split_bytes_current": 3387311104, + "inactive_split_bytes_peak": 5989206016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104400420864, + "reserved_bytes_peak": 104400420864, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T11:40:13.440314Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103328264192, + "allocated_bytes_peak_delta": 23637978112, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79690286080, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79690286080, + "device": 3, + "inactive_split_bytes_current": 4094051328, + "inactive_split_bytes_peak": 5989206016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93436510208, + "reserved_bytes_peak": 93436510208, + "segment_count_current": 510, + "segment_count_peak": 510, + "timestamp_utc": "2026-08-01T11:40:12.609911Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3387311104, + "inactive_split_bytes_current_delta": -706740224, + "inactive_split_bytes_peak": 5989206016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104400420864, + "reserved_bytes_current_delta": 10963910656, + "reserved_bytes_peak": 104400420864, + "reserved_bytes_peak_delta": 10963910656, + "segment_count_current": 587, + "segment_count_current_delta": 77, + "segment_count_peak": 587, + "segment_count_peak_delta": 77, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T114013440728Z.pickle", + "timestamp_utc": "2026-08-01T11:40:13.542909Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T114013440728Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T114013440728Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ac5969e29cb5403a839195278aa6ecf47a97bcf0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T114013440728Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:786cb6af6b0604fc8fc996262a51c4c24a4ffc7aa486f870c6f6c2f88374eef8 +size 5578069 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T114014353073Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T114014353073Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e51463256bd6f62a7cbf77ae7097b01709f69cf5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T114014353073Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103331305472, + "active_bytes_peak_delta": 3041280, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103331305472, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4008068096, + "inactive_split_bytes_peak": 5989206016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104400420864, + "reserved_bytes_peak": 104400420864, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T11:40:14.352667Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 3041280, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103328264192, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103328264192, + "device": 3, + "inactive_split_bytes_current": 3387311104, + "inactive_split_bytes_peak": 5989206016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104400420864, + "reserved_bytes_peak": 104400420864, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T11:40:13.556257Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4008068096, + "inactive_split_bytes_current_delta": 620756992, + "inactive_split_bytes_peak": 5989206016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104400420864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104400420864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 587, + "segment_count_current_delta": 0, + "segment_count_peak": 587, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T114014353073Z.pickle", + "timestamp_utc": "2026-08-01T11:40:14.454812Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T114014353073Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T114014353073Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a908c12c38f0e35298eae6daaac5959484da13a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T114014353073Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95c9b91c731481cd501d33802341ec629040058cc1da60b220bbf5b3b6a90a83 +size 5581586 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T114017033948Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T114017033948Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8299c1e407b92e52fd19429fd7c900acd23e6f1f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T114017033948Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 18144315392, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7789759488, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132422565888, + "reserved_bytes_peak": 132422565888, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:17.033437Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103331305472, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4008068096, + "inactive_split_bytes_peak": 5989206016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104400420864, + "reserved_bytes_peak": 104400420864, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T11:40:14.467791Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_current_delta": 554561536, + "inactive_split_bytes_peak": 7789759488, + "inactive_split_bytes_peak_delta": 1800553472, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132422565888, + "reserved_bytes_current_delta": 28022145024, + "reserved_bytes_peak": 132422565888, + "reserved_bytes_peak_delta": 28022145024, + "segment_count_current": 788, + "segment_count_current_delta": 201, + "segment_count_peak": 788, + "segment_count_peak_delta": 201, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T114017033948Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.153992Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T114017033948Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T114017033948Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1369b903ea36afbb4bed1cbf82497d98871455cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T114017033948Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18778e9d2f0ba3baeccd02c4ffad24f3900feda3b97bd60e6b5974b47141f33e +size 5851767 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T114017865184Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T114017865184Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31f744bef8b789a2b4315bcf6406e50fd724b691 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T114017865184Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7789759488, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132422565888, + "reserved_bytes_peak": 132422565888, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:17.864726Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7789759488, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 132422565888, + "reserved_bytes_peak": 132422565888, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:17.171108Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7789759488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132422565888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132422565888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T114017865184Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.872468Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T114017865184Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T114017865184Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb32e0b1499086850d9110e269358fa00b4a9542 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T114017865184Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46588d6d89de0890954171da6b65bef6139c0fe713ab713c2f5f6f8e1b70848b +size 241970 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T114018566243Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T114018566243Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5c1090f7674c9326b8f76882e1448886e0ee2cbd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T114018566243Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7789759488, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132422565888, + "reserved_bytes_peak": 132422565888, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:18.565827Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7789759488, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 132422565888, + "reserved_bytes_peak": 132422565888, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:17.873457Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7789759488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132422565888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132422565888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T114018566243Z.pickle", + "timestamp_utc": "2026-08-01T11:40:18.572988Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T114018566243Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T114018566243Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9cf9d10b0960d8609ce562989d4f7752b24e7332 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T114018566243Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94889c9551cdad813c70a4e4d2bcdaf3387917247503e0ac811db297a6c3b0c8 +size 241970 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T114019267956Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T114019267956Z.json new file mode 100644 index 0000000000000000000000000000000000000000..55a40aa05604d803b562a0e434489b05eb7dee65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T114019267956Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7789759488, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132422565888, + "reserved_bytes_peak": 132422565888, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:19.267563Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7789759488, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 132422565888, + "reserved_bytes_peak": 132422565888, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:18.573920Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7789759488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132422565888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132422565888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T114019267956Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.274646Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T114019267956Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T114019267956Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..abfe865b2ed4c780501492dbaf242d4735514f05 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T114019267956Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0de1b5902b6a85d8582a2599c254317a053c744005c04adf4a3f896c4b0e9f04 +size 241970 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T114019969045Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T114019969045Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34dea3d724e9f9d5a77aafe51ed1226036f577d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T114019969045Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7789759488, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132422565888, + "reserved_bytes_peak": 132422565888, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:19.968630Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7789759488, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 132422565888, + "reserved_bytes_peak": 132422565888, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:19.275615Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7789759488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132422565888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132422565888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T114019969045Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.975662Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T114019969045Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T114019969045Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d008d1d48346eec407eb7b3d3ebce729211a49a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T114019969045Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cd8dae8668f6395115ebdd800eead1501bfb64963fb73869b8c4c761dd62479 +size 241970 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T114020672957Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T114020672957Z.json new file mode 100644 index 0000000000000000000000000000000000000000..942139f9b254735281c339541e75fde051fcbc01 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T114020672957Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7789759488, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132422565888, + "reserved_bytes_peak": 132422565888, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:20.672531Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7789759488, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 132422565888, + "reserved_bytes_peak": 132422565888, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:19.976606Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7789759488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132422565888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132422565888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T114020672957Z.pickle", + "timestamp_utc": "2026-08-01T11:40:20.679740Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T114020672957Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T114020672957Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d53b6b97b31bea2ac870537308da0a18aecfc3f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T114020672957Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9134cbff4dbe7d0320bdc73fce1a20ec1f69fa1f61426c164085a8e6b02d277 +size 241970 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T114113364317Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T114113364317Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2012d087bf8da11ebc39cbf78c6edd33e1ad5bee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T114113364317Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855801856, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855801856, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66855801856, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 2585095168, + "inactive_split_bytes_peak": 7789759488, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132445634560, + "reserved_bytes_peak": 132445634560, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:41:13.363823Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855801856, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 3, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7789759488, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 132422565888, + "reserved_bytes_peak": 132422565888, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:20.680680Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2585095168, + "inactive_split_bytes_current_delta": -1977534464, + "inactive_split_bytes_peak": 7789759488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132445634560, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132445634560, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 790, + "segment_count_current_delta": 2, + "segment_count_peak": 790, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T114113364317Z.pickle", + "timestamp_utc": "2026-08-01T11:41:13.446892Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T114113364317Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T114113364317Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b8eb413093b574e0ee4451dc3611353df45d91dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T114113364317Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19a9753804c175509708e508dc7fe0208e5444f8f9614d9f273bd00cd969a4d3 +size 4163113 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T114012483708Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T114012483708Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c78d9425f25b5feee2060ec906079585299726f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T114012483708Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872895488, + "active_bytes_current_delta": 31640997888, + "active_bytes_peak": 79688975360, + "active_bytes_peak_delta": 44222196736, + "after": { + "active_bytes_current": 66872895488, + "active_bytes_peak": 79688975360, + "allocated_bytes_current": 66872895488, + "allocated_bytes_peak": 79688975360, + "device": 4, + "inactive_split_bytes_current": 4050688000, + "inactive_split_bytes_peak": 5154283520, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93457481728, + "reserved_bytes_peak": 93457481728, + "segment_count_current": 511, + "segment_count_peak": 511, + "timestamp_utc": "2026-08-01T11:40:12.483238Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872895488, + "allocated_bytes_current_delta": 31640997888, + "allocated_bytes_peak": 79688975360, + "allocated_bytes_peak_delta": 44222196736, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:39:04.476496Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4050688000, + "inactive_split_bytes_current_delta": 3918311424, + "inactive_split_bytes_peak": 5154283520, + "inactive_split_bytes_peak_delta": 5000873984, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93457481728, + "reserved_bytes_current_delta": 54213476352, + "reserved_bytes_peak": 93457481728, + "reserved_bytes_peak_delta": 54213476352, + "segment_count_current": 511, + "segment_count_current_delta": 440, + "segment_count_peak": 511, + "segment_count_peak_delta": 440, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T114012483708Z.pickle", + "timestamp_utc": "2026-08-01T11:40:12.598956Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T114012483708Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T114012483708Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..687b291d32d235cbb9e386502659db9c0f2ad15f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T114012483708Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83c1e20b26d3293812f51d9258aa61a394bef1aa0b7b9e1cc93d4569a38fa741 +size 5380938 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T114013440642Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T114013440642Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee9f8891e1d32d0b3a4a67176484cadfb80537e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T114013440642Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872895488, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103323623424, + "active_bytes_peak_delta": 23634648064, + "after": { + "active_bytes_current": 66872895488, + "active_bytes_peak": 103323623424, + "allocated_bytes_current": 66872895488, + "allocated_bytes_peak": 103323623424, + "device": 4, + "inactive_split_bytes_current": 3327170560, + "inactive_split_bytes_peak": 5154283520, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104413003776, + "reserved_bytes_peak": 104413003776, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T11:40:13.440239Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872895488, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103323623424, + "allocated_bytes_peak_delta": 23634648064, + "before": { + "active_bytes_current": 66872895488, + "active_bytes_peak": 79688975360, + "allocated_bytes_current": 66872895488, + "allocated_bytes_peak": 79688975360, + "device": 4, + "inactive_split_bytes_current": 4050688000, + "inactive_split_bytes_peak": 5154283520, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93457481728, + "reserved_bytes_peak": 93457481728, + "segment_count_current": 511, + "segment_count_peak": 511, + "timestamp_utc": "2026-08-01T11:40:12.618772Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3327170560, + "inactive_split_bytes_current_delta": -723517440, + "inactive_split_bytes_peak": 5154283520, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104413003776, + "reserved_bytes_current_delta": 10955522048, + "reserved_bytes_peak": 104413003776, + "reserved_bytes_peak_delta": 10955522048, + "segment_count_current": 587, + "segment_count_current_delta": 76, + "segment_count_peak": 587, + "segment_count_peak_delta": 76, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T114013440642Z.pickle", + "timestamp_utc": "2026-08-01T11:40:13.545456Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T114013440642Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T114013440642Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fd3bdb178d071298c283feb81cfc315ad82ecd4b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T114013440642Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37cc972b60f134769f576dca36c8e0243c17dd7602efa40eab5d5f5d173a6b79 +size 5585560 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T114014353044Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T114014353044Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1237efb109c0f2ba8a78a4fe83bd30f4adc449e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T114014353044Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872895488, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103323623424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66872895488, + "active_bytes_peak": 103323623424, + "allocated_bytes_current": 66872895488, + "allocated_bytes_peak": 103323623424, + "device": 4, + "inactive_split_bytes_current": 4088436736, + "inactive_split_bytes_peak": 5348102656, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104413003776, + "reserved_bytes_peak": 104413003776, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T11:40:14.352646Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872895488, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103323623424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66872895488, + "active_bytes_peak": 103323623424, + "allocated_bytes_current": 66872895488, + "allocated_bytes_peak": 103323623424, + "device": 4, + "inactive_split_bytes_current": 3327170560, + "inactive_split_bytes_peak": 5154283520, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104413003776, + "reserved_bytes_peak": 104413003776, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T11:40:13.560662Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4088436736, + "inactive_split_bytes_current_delta": 761266176, + "inactive_split_bytes_peak": 5348102656, + "inactive_split_bytes_peak_delta": 193819136, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104413003776, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104413003776, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 587, + "segment_count_current_delta": 0, + "segment_count_peak": 587, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T114014353044Z.pickle", + "timestamp_utc": "2026-08-01T11:40:14.455429Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T114014353044Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T114014353044Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..22ca384a2c9efd293fc2bc88d45e2fb165a01d70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T114014353044Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d537f95788e32091734967b8df5061971b531afb3a59ff9d0fbc99fd6edb802c +size 5588664 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T114017033906Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T114017033906Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6b041a9695e28a6d99a21916515b2b2da1346df5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T114017033906Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890856448, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 121478876160, + "active_bytes_peak_delta": 18155252736, + "after": { + "active_bytes_current": 66890856448, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66890856448, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132504354816, + "reserved_bytes_peak": 132504354816, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:17.033360Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890856448, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103324068864, + "allocated_bytes_peak_delta": 445440, + "before": { + "active_bytes_current": 66872895488, + "active_bytes_peak": 103323623424, + "allocated_bytes_current": 66872895488, + "allocated_bytes_peak": 103323623424, + "device": 4, + "inactive_split_bytes_current": 4088436736, + "inactive_split_bytes_peak": 5348102656, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104413003776, + "reserved_bytes_peak": 104413003776, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T11:40:14.469962Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_current_delta": 455995392, + "inactive_split_bytes_peak": 8125980672, + "inactive_split_bytes_peak_delta": 2777878016, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132504354816, + "reserved_bytes_current_delta": 28091351040, + "reserved_bytes_peak": 132504354816, + "reserved_bytes_peak_delta": 28091351040, + "segment_count_current": 790, + "segment_count_current_delta": 203, + "segment_count_peak": 790, + "segment_count_peak_delta": 203, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T114017033906Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.152112Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T114017033906Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T114017033906Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..147ec9d6496e8ec096e4ffb39e70bbbfd7444770 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T114017033906Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbaa38f27f552acc9c4b8c606c8d69d57c49f8d964d2a13256413ab538652771 +size 5861420 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T114017865105Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T114017865105Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2c646ab9333b70252eb526e9ac805a1a5e322b48 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T114017865105Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890856448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478876160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890856448, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66890856448, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132504354816, + "reserved_bytes_peak": 132504354816, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:17.864643Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890856448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103324068864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890856448, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66890856448, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 132504354816, + "reserved_bytes_peak": 132504354816, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:17.168416Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8125980672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132504354816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132504354816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 790, + "segment_count_current_delta": 0, + "segment_count_peak": 790, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T114017865105Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.872690Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T114017865105Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T114017865105Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..664eda064a6e5ffa263042079855d6c27baf6edc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T114017865105Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdffe8062bd063c5d76e7e58a4307f4eed0084b6a1b722bcf3f8f46718b76c59 +size 250503 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T114018566193Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T114018566193Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c615fa14df0180e294fe153e9dcadc8f64dea0ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T114018566193Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890856448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478876160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890856448, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66890856448, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132504354816, + "reserved_bytes_peak": 132504354816, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:18.565790Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890856448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103324068864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890856448, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66890856448, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 132504354816, + "reserved_bytes_peak": 132504354816, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:17.873665Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8125980672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132504354816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132504354816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 790, + "segment_count_current_delta": 0, + "segment_count_peak": 790, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T114018566193Z.pickle", + "timestamp_utc": "2026-08-01T11:40:18.572987Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T114018566193Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T114018566193Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff9656dfdc3541e25132bae9feecd214de052312 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T114018566193Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcc147237c79490905a750df790dd3b0fa968251d721a059244847d366a0fef2 +size 250503 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T114019267923Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T114019267923Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9996d177db4e04ba898e06da3bf75509c0c3150 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T114019267923Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890856448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478876160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890856448, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66890856448, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132504354816, + "reserved_bytes_peak": 132504354816, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:19.267530Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890856448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103324068864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890856448, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66890856448, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 132504354816, + "reserved_bytes_peak": 132504354816, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:18.573960Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8125980672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132504354816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132504354816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 790, + "segment_count_current_delta": 0, + "segment_count_peak": 790, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T114019267923Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.274382Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T114019267923Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T114019267923Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b2680e541b5ad61f8d19c812b80967ed0f17fbd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T114019267923Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66199c6768ee3a48a3eba870ab831fac2e967bf4888edaae5491b3b640868659 +size 250503 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T114019968991Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T114019968991Z.json new file mode 100644 index 0000000000000000000000000000000000000000..19fcb456e9f48fa690ab8417772a5cb971463f5e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T114019968991Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890856448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478876160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890856448, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66890856448, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132504354816, + "reserved_bytes_peak": 132504354816, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:19.968593Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890856448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103324068864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890856448, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66890856448, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 132504354816, + "reserved_bytes_peak": 132504354816, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:19.275369Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8125980672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132504354816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132504354816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 790, + "segment_count_current_delta": 0, + "segment_count_peak": 790, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T114019968991Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.975602Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T114019968991Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T114019968991Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8bb91bd927f6f24a877bfc150ef3eb54e7a18361 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T114019968991Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60239ec8a80bb005b01cccdfbd4ff57f6a0a5edb3b6efac0087a7a87b55be9e6 +size 250503 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T114020672866Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T114020672866Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0b279d11c62c3c10b71ca2be41362834b78d3cd2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T114020672866Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890856448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121478876160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890856448, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66890856448, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132504354816, + "reserved_bytes_peak": 132504354816, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:20.672478Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890856448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103324068864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890856448, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66890856448, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 132504354816, + "reserved_bytes_peak": 132504354816, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:19.976526Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8125980672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132504354816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132504354816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 790, + "segment_count_current_delta": 0, + "segment_count_peak": 790, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T114020672866Z.pickle", + "timestamp_utc": "2026-08-01T11:40:20.679572Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T114020672866Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T114020672866Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dd114f31c064cf65c6ae23a3243310ec40982ecc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T114020672866Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90108d7127a1ced48b0f3d43b657f61c370c647dd07e08cdd85b531cc2b86ba5 +size 250503 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T114113364325Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T114113364325Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a56a297d52e9c0786af91239300c4c7c9d6e8c5b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T114113364325Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66857222144, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 121478876160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66857222144, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66857222144, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 2476720128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132527423488, + "reserved_bytes_peak": 132527423488, + "segment_count_current": 792, + "segment_count_peak": 792, + "timestamp_utc": "2026-08-01T11:41:13.363835Z", + "world_size": 8 + }, + "allocated_bytes_current": 66857222144, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103324068864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890856448, + "active_bytes_peak": 121478876160, + "allocated_bytes_current": 66890856448, + "allocated_bytes_peak": 103324068864, + "device": 4, + "inactive_split_bytes_current": 4544432128, + "inactive_split_bytes_peak": 8125980672, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 132504354816, + "reserved_bytes_peak": 132504354816, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:40:20.680526Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2476720128, + "inactive_split_bytes_current_delta": -2067712000, + "inactive_split_bytes_peak": 8125980672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132527423488, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132527423488, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 792, + "segment_count_current_delta": 2, + "segment_count_peak": 792, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T114113364325Z.pickle", + "timestamp_utc": "2026-08-01T11:41:13.448768Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T114113364325Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T114113364325Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d09a08c061d2f489836fa1c19c00f06be45109b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T114113364325Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56a7af4cb8e444934b05439ceab45205e9dd531e1142be67df19f69c2630fd24 +size 4183462 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T114012483709Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T114012483709Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f7f38d6b5cd01444521b299d303b1993281f2a32 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T114012483709Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 31640019968, + "active_bytes_peak": 79690286080, + "active_bytes_peak_delta": 44223949824, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79690286080, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79690286080, + "device": 5, + "inactive_split_bytes_current": 4094051328, + "inactive_split_bytes_peak": 5871921152, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93436510208, + "reserved_bytes_peak": 93436510208, + "segment_count_current": 510, + "segment_count_peak": 510, + "timestamp_utc": "2026-08-01T11:40:12.483111Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 31640019968, + "allocated_bytes_peak": 79690286080, + "allocated_bytes_peak_delta": 44223949824, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:39:04.476601Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4094051328, + "inactive_split_bytes_current_delta": 3961232384, + "inactive_split_bytes_peak": 5871921152, + "inactive_split_bytes_peak_delta": 5711535104, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93436510208, + "reserved_bytes_current_delta": 54192504832, + "reserved_bytes_peak": 93436510208, + "reserved_bytes_peak_delta": 54192504832, + "segment_count_current": 510, + "segment_count_current_delta": 439, + "segment_count_peak": 510, + "segment_count_peak_delta": 439, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T114012483709Z.pickle", + "timestamp_utc": "2026-08-01T11:40:12.594402Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T114012483709Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T114012483709Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..91e14d93055dcfd8d33112849028fab4b8b1d74f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T114012483709Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c14225ae1aa468a89d2e6d8ed7696e9ce8388977bef75b42343c8fe7cbdf04d +size 5375046 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T114013440635Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T114013440635Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f8459fd239d527dbc7e7e254f561a29b4cd3cea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T114013440635Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103328264192, + "active_bytes_peak_delta": 23637978112, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103328264192, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103328264192, + "device": 5, + "inactive_split_bytes_current": 3387311104, + "inactive_split_bytes_peak": 5871921152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104400420864, + "reserved_bytes_peak": 104400420864, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T11:40:13.440229Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103328264192, + "allocated_bytes_peak_delta": 23637978112, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79690286080, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79690286080, + "device": 5, + "inactive_split_bytes_current": 4094051328, + "inactive_split_bytes_peak": 5871921152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93436510208, + "reserved_bytes_peak": 93436510208, + "segment_count_current": 510, + "segment_count_peak": 510, + "timestamp_utc": "2026-08-01T11:40:12.613862Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3387311104, + "inactive_split_bytes_current_delta": -706740224, + "inactive_split_bytes_peak": 5871921152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104400420864, + "reserved_bytes_current_delta": 10963910656, + "reserved_bytes_peak": 104400420864, + "reserved_bytes_peak_delta": 10963910656, + "segment_count_current": 587, + "segment_count_current_delta": 77, + "segment_count_peak": 587, + "segment_count_peak_delta": 77, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T114013440635Z.pickle", + "timestamp_utc": "2026-08-01T11:40:13.545969Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T114013440635Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T114013440635Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a11af8a2a744e563ff08be1ceed2ab8291dba95 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T114013440635Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca44b4f5e9f475978dce908188a5f0c8e0bad8d06692d78740aaa0b03cfa569c +size 5576089 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T114014353031Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T114014353031Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04e8b0e3960d90842f3ea1ec437d8ae26aaf56c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T114014353031Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103331305472, + "active_bytes_peak_delta": 3041280, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103331305472, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4010165248, + "inactive_split_bytes_peak": 5871921152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104400420864, + "reserved_bytes_peak": 104400420864, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T11:40:14.352644Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 3041280, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103328264192, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103328264192, + "device": 5, + "inactive_split_bytes_current": 3387311104, + "inactive_split_bytes_peak": 5871921152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104400420864, + "reserved_bytes_peak": 104400420864, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T11:40:13.561193Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4010165248, + "inactive_split_bytes_current_delta": 622854144, + "inactive_split_bytes_peak": 5871921152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104400420864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104400420864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 587, + "segment_count_current_delta": 0, + "segment_count_peak": 587, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T114014353031Z.pickle", + "timestamp_utc": "2026-08-01T11:40:14.456541Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T114014353031Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T114014353031Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13779867bd1b462cbf81ad1cec4be7fc0b409775 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T114014353031Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b219dac0ec1e2227e86ec91cfe3a6c36cbe765d148022bc0167b01f098b916a1 +size 5579291 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T114017033918Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T114017033918Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f979ad8d597a20e4725bcac30e68a1c6a7de8602 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T114017033918Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 18144315392, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7787662336, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132424663040, + "reserved_bytes_peak": 132424663040, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:17.033381Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103331305472, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4010165248, + "inactive_split_bytes_peak": 5871921152, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104400420864, + "reserved_bytes_peak": 104400420864, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T11:40:14.471325Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_current_delta": 552464384, + "inactive_split_bytes_peak": 7787662336, + "inactive_split_bytes_peak_delta": 1915741184, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132424663040, + "reserved_bytes_current_delta": 28024242176, + "reserved_bytes_peak": 132424663040, + "reserved_bytes_peak_delta": 28024242176, + "segment_count_current": 789, + "segment_count_current_delta": 202, + "segment_count_peak": 789, + "segment_count_peak_delta": 202, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T114017033918Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.154256Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T114017033918Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T114017033918Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd14d3cee536752d59583345d11846da165383c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T114017033918Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e4a1fe20924defa51b2084cd4be38b0b5eb634867f9615e8c9f14cdc14e2aeb +size 5849850 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T114017865103Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T114017865103Z.json new file mode 100644 index 0000000000000000000000000000000000000000..430fc5ec61deb5b55fe879237faa38a144782e53 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T114017865103Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7787662336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132424663040, + "reserved_bytes_peak": 132424663040, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:17.864651Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7787662336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 132424663040, + "reserved_bytes_peak": 132424663040, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:17.170639Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7787662336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132424663040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132424663040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 789, + "segment_count_current_delta": 0, + "segment_count_peak": 789, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T114017865103Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.872393Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T114017865103Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T114017865103Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a1770b9ca367b180eeeec0cb8f682b9605c8f66c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T114017865103Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:635057514d6fdaefc3b8ebc1f669d4f4a16867f6c9bfd16cf76e989057285cba +size 240190 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T114018566368Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T114018566368Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ff665c5b33e161707cce27aaf561f27c10bf318d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T114018566368Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7787662336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132424663040, + "reserved_bytes_peak": 132424663040, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:18.565951Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7787662336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 132424663040, + "reserved_bytes_peak": 132424663040, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:17.873379Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7787662336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132424663040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132424663040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 789, + "segment_count_current_delta": 0, + "segment_count_peak": 789, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T114018566368Z.pickle", + "timestamp_utc": "2026-08-01T11:40:18.575235Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T114018566368Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T114018566368Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9fb3870faa2d7c531e0e6803a160e80303b1df3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T114018566368Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c7fd7f33fbfe9bdc859ca2cb52e7815ba51de61713dc56b0fe137c5ff3c01ae +size 240190 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T114019267953Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T114019267953Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ca77dcdeb210db7124a19f394689d0ad157e1b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T114019267953Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7787662336, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132424663040, + "reserved_bytes_peak": 132424663040, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:19.267556Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7787662336, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 132424663040, + "reserved_bytes_peak": 132424663040, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:18.576199Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7787662336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132424663040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132424663040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 789, + "segment_count_current_delta": 0, + "segment_count_peak": 789, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T114019267953Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.274359Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T114019267953Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T114019267953Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d65d67d397eb3acafbec62b168f8766a4def8fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T114019267953Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce321203d2049c950af2ef70b98d3d1f21b7677eda7dc83305e0a45635673d49 +size 240190 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T114019969006Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T114019969006Z.json new file mode 100644 index 0000000000000000000000000000000000000000..41e1ab99e10437e07a403e633ba23999ff159030 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T114019969006Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7787662336, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132424663040, + "reserved_bytes_peak": 132424663040, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:19.968609Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7787662336, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 132424663040, + "reserved_bytes_peak": 132424663040, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:19.275300Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7787662336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132424663040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132424663040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 789, + "segment_count_current_delta": 0, + "segment_count_peak": 789, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T114019969006Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.979023Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T114019969006Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T114019969006Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba8fbeb3d4053c5e957ee7cae40db5595d891d27 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T114019969006Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8201da94e4bc77ebc7d2c412ab197e22cc82f91ae069afb54c383cee972987b8 +size 240190 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T114020672901Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T114020672901Z.json new file mode 100644 index 0000000000000000000000000000000000000000..14ded0ef561e382f5dc9ccc2ab74f56cd2e2648d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T114020672901Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7787662336, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132424663040, + "reserved_bytes_peak": 132424663040, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:20.672498Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7787662336, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 132424663040, + "reserved_bytes_peak": 132424663040, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:19.980001Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7787662336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132424663040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132424663040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 789, + "segment_count_current_delta": 0, + "segment_count_peak": 789, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T114020672901Z.pickle", + "timestamp_utc": "2026-08-01T11:40:20.679567Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T114020672901Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T114020672901Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03d1d083bb74831a3ed5e60e9fe55b50c0e4d466 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T114020672901Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:284bf3cd8da2d94c5fc525f08cd3570eb841b45b774771fc03b62ec59b48ee68 +size 240190 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T114113364311Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T114113364311Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9bb95897fea138a632713eed3f0551011b4e9663 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T114113364311Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855801856, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 121475620864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855801856, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66855801856, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 2585095168, + "inactive_split_bytes_peak": 7787662336, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132447731712, + "reserved_bytes_peak": 132447731712, + "segment_count_current": 791, + "segment_count_peak": 791, + "timestamp_utc": "2026-08-01T11:41:13.363829Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855801856, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103331305472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 121475620864, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103331305472, + "device": 5, + "inactive_split_bytes_current": 4562629632, + "inactive_split_bytes_peak": 7787662336, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 132424663040, + "reserved_bytes_peak": 132424663040, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:20.680494Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2585095168, + "inactive_split_bytes_current_delta": -1977534464, + "inactive_split_bytes_peak": 7787662336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132447731712, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132447731712, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 791, + "segment_count_current_delta": 2, + "segment_count_peak": 791, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T114113364311Z.pickle", + "timestamp_utc": "2026-08-01T11:41:13.447651Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T114113364311Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T114113364311Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0f18510a6bec4179748fe6c5e3c31f05fec2dc3b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T114113364311Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:085b353c202df14eb387563e9f0650aed3ce84278e5b917263632b8f491df99d +size 4120967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T114012483813Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T114012483813Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3af6db78c7a11be975598b3d8fa794aacd6cccf4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T114012483813Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 31640019968, + "active_bytes_peak": 79690286080, + "active_bytes_peak_delta": 44223949824, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79690286080, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79690286080, + "device": 6, + "inactive_split_bytes_current": 4328932352, + "inactive_split_bytes_peak": 5463897088, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93906272256, + "reserved_bytes_peak": 93906272256, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T11:40:12.483179Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 31640019968, + "allocated_bytes_peak": 79690286080, + "allocated_bytes_peak_delta": 44223949824, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:39:04.476604Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4328932352, + "inactive_split_bytes_current_delta": 4196113408, + "inactive_split_bytes_peak": 5463897088, + "inactive_split_bytes_peak_delta": 5303511040, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93906272256, + "reserved_bytes_current_delta": 54662266880, + "reserved_bytes_peak": 93906272256, + "reserved_bytes_peak_delta": 54662266880, + "segment_count_current": 517, + "segment_count_current_delta": 446, + "segment_count_peak": 517, + "segment_count_peak_delta": 446, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T114012483813Z.pickle", + "timestamp_utc": "2026-08-01T11:40:12.593377Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T114012483813Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T114012483813Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a1cbd92bcbdce2ebb6f4563d971619dbec25d296 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T114012483813Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57949760b57c38a32d6e6847124ffa175d359c7c31444bc82c66c735faef16f7 +size 5378636 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T114013440718Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T114013440718Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9bf160384971dffe34231848c014ed1536e9ae9e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T114013440718Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103326691328, + "active_bytes_peak_delta": 23636405248, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103326691328, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103326691328, + "device": 6, + "inactive_split_bytes_current": 4024845312, + "inactive_split_bytes_peak": 5463897088, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 589, + "segment_count_peak": 589, + "timestamp_utc": "2026-08-01T11:40:13.440294Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691328, + "allocated_bytes_peak_delta": 23636405248, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79690286080, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79690286080, + "device": 6, + "inactive_split_bytes_current": 4328932352, + "inactive_split_bytes_peak": 5463897088, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93906272256, + "reserved_bytes_peak": 93906272256, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T11:40:12.611006Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4024845312, + "inactive_split_bytes_current_delta": -304087040, + "inactive_split_bytes_peak": 5463897088, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104421392384, + "reserved_bytes_current_delta": 10515120128, + "reserved_bytes_peak": 104421392384, + "reserved_bytes_peak_delta": 10515120128, + "segment_count_current": 589, + "segment_count_current_delta": 72, + "segment_count_peak": 589, + "segment_count_peak_delta": 72, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T114013440718Z.pickle", + "timestamp_utc": "2026-08-01T11:40:13.543353Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T114013440718Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T114013440718Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e8e8ea7e24eaaf0ef67fbb80cb77e5bac2c51a57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T114013440718Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a93b15f4c6767ab330f3515959a0ac4324b66ea0879e8d9618e8aad4670ebd1 +size 5580238 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T114014353070Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T114014353070Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca04d4cf9bbce6c0009455f2a451bc1171b0a36c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T114014353070Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103329097728, + "active_bytes_peak_delta": 2406400, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103329097728, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 3737535488, + "inactive_split_bytes_peak": 5463897088, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 589, + "segment_count_peak": 589, + "timestamp_utc": "2026-08-01T11:40:14.352680Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 2406400, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103326691328, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103326691328, + "device": 6, + "inactive_split_bytes_current": 4024845312, + "inactive_split_bytes_peak": 5463897088, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 589, + "segment_count_peak": 589, + "timestamp_utc": "2026-08-01T11:40:13.556737Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3737535488, + "inactive_split_bytes_current_delta": -287309824, + "inactive_split_bytes_peak": 5463897088, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104421392384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104421392384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 589, + "segment_count_current_delta": 0, + "segment_count_peak": 589, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T114014353070Z.pickle", + "timestamp_utc": "2026-08-01T11:40:14.452682Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T114014353070Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T114014353070Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e910bf8ffae370a17a16cf8647225f1aa8617b13 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T114014353070Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e0ecc7578ed9865f317b14e2988ecd356b7ae5c7da66b31e4cfc8e6d49b3153 +size 5579126 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T114017033938Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T114017033938Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c66a276add8da52271dfde9763ab70b552289efa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T114017033938Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 18148358144, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_peak": 7709412352, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132126867456, + "reserved_bytes_peak": 132126867456, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:17.033407Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103329097728, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 3737535488, + "inactive_split_bytes_peak": 5463897088, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 589, + "segment_count_peak": 589, + "timestamp_utc": "2026-08-01T11:40:14.465670Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_current_delta": 389804032, + "inactive_split_bytes_peak": 7709412352, + "inactive_split_bytes_peak_delta": 2245515264, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132126867456, + "reserved_bytes_current_delta": 27705475072, + "reserved_bytes_peak": 132126867456, + "reserved_bytes_peak_delta": 27705475072, + "segment_count_current": 789, + "segment_count_current_delta": 200, + "segment_count_peak": 789, + "segment_count_peak_delta": 200, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T114017033938Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.153588Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T114017033938Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T114017033938Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1210be582a4a4b3610ef9021b139fc0cedd3b8cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T114017033938Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dc0bfdf17514f2caf644b282e14b5cc3d078b4c15e805ac0e1e12c9aadf5432 +size 5851865 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T114017865172Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T114017865172Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8cb23bfec07e46aa5ea73187a8031dafd19e7bed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T114017865172Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_peak": 7709412352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132126867456, + "reserved_bytes_peak": 132126867456, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:17.864710Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_peak": 7709412352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132126867456, + "reserved_bytes_peak": 132126867456, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:17.170482Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7709412352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132126867456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132126867456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 789, + "segment_count_current_delta": 0, + "segment_count_peak": 789, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T114017865172Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.872463Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T114017865172Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T114017865172Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9425cea767beac43506f36848c8d7085fd8351c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T114017865172Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc984e454b9bfdb2d4d1a4d8fb62297c58c635e4d214bb65edea0bfdeeac66dc +size 243236 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T114018566243Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T114018566243Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0059c7f138f60eb45b7721e0e5e74f54b8c62212 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T114018566243Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_peak": 7709412352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132126867456, + "reserved_bytes_peak": 132126867456, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:18.565830Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_peak": 7709412352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132126867456, + "reserved_bytes_peak": 132126867456, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:17.873495Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7709412352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132126867456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132126867456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 789, + "segment_count_current_delta": 0, + "segment_count_peak": 789, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T114018566243Z.pickle", + "timestamp_utc": "2026-08-01T11:40:18.572826Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T114018566243Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T114018566243Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f6afee0ba4228e69847c04be6627b487bfc7e09c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T114018566243Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:797855c621437895cebd81e4618ed16d19f2abddeddcb06e2fcdbcd7b2ef57b2 +size 243236 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T114019267996Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T114019267996Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f0d1afe215af6207112bf2ebec8757478fd58a25 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T114019267996Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_peak": 7709412352, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132126867456, + "reserved_bytes_peak": 132126867456, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:19.267595Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_peak": 7709412352, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132126867456, + "reserved_bytes_peak": 132126867456, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:18.573775Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7709412352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132126867456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132126867456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 789, + "segment_count_current_delta": 0, + "segment_count_peak": 789, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T114019267996Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.274381Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T114019267996Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T114019267996Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..19fc4abe1a25ba138fe873efc5c4df88ff2fd5a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T114019267996Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d42538496ad633d6080bb59b179006c702556d772a89b3aef8ad671b7417107 +size 243236 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T114019969032Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T114019969032Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6ef07c3890ad986aec51bcac5a6d6b635824b42 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T114019969032Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_peak": 7709412352, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132126867456, + "reserved_bytes_peak": 132126867456, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:19.968634Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_peak": 7709412352, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132126867456, + "reserved_bytes_peak": 132126867456, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:19.275338Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7709412352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132126867456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132126867456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 789, + "segment_count_current_delta": 0, + "segment_count_peak": 789, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T114019969032Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.975305Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T114019969032Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T114019969032Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..34c372bce5fb1b90ab8732c303c10e624f26284b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T114019969032Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:556d8749352cfafd9d64bbe489b0bd93461b861200e8c66ad2a4a9b1d9e53ecf +size 243236 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T114020673035Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T114020673035Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef38d5a3754bf75c5d7d2525acb293266f6009b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T114020673035Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888518656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_peak": 7709412352, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132126867456, + "reserved_bytes_peak": 132126867456, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:20.672577Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888518656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_peak": 7709412352, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132126867456, + "reserved_bytes_peak": 132126867456, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:19.976270Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7709412352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132126867456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132126867456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 789, + "segment_count_current_delta": 0, + "segment_count_peak": 789, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T114020673035Z.pickle", + "timestamp_utc": "2026-08-01T11:40:20.679971Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T114020673035Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T114020673035Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..30aedb8e42419e796b9f714aee32fd98f62b3813 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T114020673035Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a22d0fc0aa4b0059422c3a0bd31f4254af05d0313c10e84ab38b8479f3af44e2 +size 243236 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T114113364279Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T114113364279Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31cb2041d4ddf7572db956248a34925c76a32bc2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T114113364279Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854884352, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 121477455872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854884352, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66854884352, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 2296605696, + "inactive_split_bytes_peak": 7709412352, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132149936128, + "reserved_bytes_peak": 132149936128, + "segment_count_current": 791, + "segment_count_peak": 791, + "timestamp_utc": "2026-08-01T11:41:13.363814Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854884352, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103329097728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888518656, + "active_bytes_peak": 121477455872, + "allocated_bytes_current": 66888518656, + "allocated_bytes_peak": 103329097728, + "device": 6, + "inactive_split_bytes_current": 4127339520, + "inactive_split_bytes_peak": 7709412352, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132126867456, + "reserved_bytes_peak": 132126867456, + "segment_count_current": 789, + "segment_count_peak": 789, + "timestamp_utc": "2026-08-01T11:40:20.680939Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2296605696, + "inactive_split_bytes_current_delta": -1830733824, + "inactive_split_bytes_peak": 7709412352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132149936128, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132149936128, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 791, + "segment_count_current_delta": 2, + "segment_count_peak": 791, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T114113364279Z.pickle", + "timestamp_utc": "2026-08-01T11:41:13.447659Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T114113364279Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T114113364279Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a781c67e890b609e6f6851b8fd95bbd9e63a8ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T114113364279Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfb34f32e57c31e3aa9b58d008cfe117bd800074876bea798f2ffb0d76ea9295 +size 4121879 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T114012483676Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T114012483676Z.json new file mode 100644 index 0000000000000000000000000000000000000000..60cf3e879cdf34fd524aa4df9138865f7c91faca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T114012483676Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475712, + "active_bytes_current_delta": 31640020480, + "active_bytes_peak": 79690286080, + "active_bytes_peak_delta": 44223949824, + "after": { + "active_bytes_current": 66871475712, + "active_bytes_peak": 79690286080, + "allocated_bytes_current": 66871475712, + "allocated_bytes_peak": 79690286080, + "device": 7, + "inactive_split_bytes_current": 4331028992, + "inactive_split_bytes_peak": 5231113216, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 94376034304, + "reserved_bytes_peak": 94376034304, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T11:40:12.483139Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475712, + "allocated_bytes_current_delta": 31640020480, + "allocated_bytes_peak": 79690286080, + "allocated_bytes_peak_delta": 44223949824, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:39:04.476630Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4331028992, + "inactive_split_bytes_current_delta": 4198210048, + "inactive_split_bytes_peak": 5231113216, + "inactive_split_bytes_peak_delta": 5070727168, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 94376034304, + "reserved_bytes_current_delta": 55132028928, + "reserved_bytes_peak": 94376034304, + "reserved_bytes_peak_delta": 55132028928, + "segment_count_current": 520, + "segment_count_current_delta": 449, + "segment_count_peak": 520, + "segment_count_peak_delta": 449, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T114012483676Z.pickle", + "timestamp_utc": "2026-08-01T11:40:12.595595Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T114012483676Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T114012483676Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf7b50911d0d6be680053fec447c75f443020c94 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T114012483676Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4534e9a15fa0605be6f9e2a8adcee09393f1f4fceb9d8edf6133b9bf147aa1a9 +size 5379621 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T114013440745Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T114013440745Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eb4cdd0d16a47b33c8d7012deb65f59d8941344a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T114013440745Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103326691840, + "active_bytes_peak_delta": 23636405760, + "after": { + "active_bytes_current": 66871475712, + "active_bytes_peak": 103326691840, + "allocated_bytes_current": 66871475712, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 3624288768, + "inactive_split_bytes_peak": 5231113216, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T11:40:13.440327Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691840, + "allocated_bytes_peak_delta": 23636405760, + "before": { + "active_bytes_current": 66871475712, + "active_bytes_peak": 79690286080, + "allocated_bytes_current": 66871475712, + "allocated_bytes_peak": 79690286080, + "device": 7, + "inactive_split_bytes_current": 4331028992, + "inactive_split_bytes_peak": 5231113216, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 94376034304, + "reserved_bytes_peak": 94376034304, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T11:40:12.615132Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3624288768, + "inactive_split_bytes_current_delta": -706740224, + "inactive_split_bytes_peak": 5231113216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104421392384, + "reserved_bytes_current_delta": 10045358080, + "reserved_bytes_peak": 104421392384, + "reserved_bytes_peak_delta": 10045358080, + "segment_count_current": 590, + "segment_count_current_delta": 70, + "segment_count_peak": 590, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T114013440745Z.pickle", + "timestamp_utc": "2026-08-01T11:40:13.546509Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T114013440745Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T114013440745Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94bae214dc36c45c28842d91e4b69b8074ec09a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T114013440745Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7bdea3518dcc3e9c3c4bb3ac757885d00e4cbd2c960c2d3b5cd77c950978fc5 +size 5582113 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T114014353109Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T114014353109Z.json new file mode 100644 index 0000000000000000000000000000000000000000..63f1aec99ea6a868582861d4ca4986e078707810 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T114014353109Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103326691840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871475712, + "active_bytes_peak": 103326691840, + "allocated_bytes_current": 66871475712, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 3708174848, + "inactive_split_bytes_peak": 5231113216, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T11:40:14.352704Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475712, + "active_bytes_peak": 103326691840, + "allocated_bytes_current": 66871475712, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 3624288768, + "inactive_split_bytes_peak": 5231113216, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T11:40:13.561754Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3708174848, + "inactive_split_bytes_current_delta": 83886080, + "inactive_split_bytes_peak": 5231113216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104421392384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104421392384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T114014353109Z.pickle", + "timestamp_utc": "2026-08-01T11:40:14.457992Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T114014353109Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T114014353109Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d95ffe1b5d919308ce46e8e83e0f2005e9d6906e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T114014353109Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acda5c4a8f5353533d03784dff753bb91394d1943ddb59d981231303d1e3384e +size 5583694 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T114017033967Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T114017033967Z.json new file mode 100644 index 0000000000000000000000000000000000000000..594e51d5ddd90ed7890ca5ab3e99d3443efa9b13 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T114017033967Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888519168, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121477456384, + "active_bytes_peak_delta": 18150764544, + "after": { + "active_bytes_current": 66888519168, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66888519168, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 4248973824, + "inactive_split_bytes_peak": 7690537472, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132097507328, + "reserved_bytes_peak": 132097507328, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:17.033452Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888519168, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103326691840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475712, + "active_bytes_peak": 103326691840, + "allocated_bytes_current": 66871475712, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 3708174848, + "inactive_split_bytes_peak": 5231113216, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104421392384, + "reserved_bytes_peak": 104421392384, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T11:40:14.472942Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4248973824, + "inactive_split_bytes_current_delta": 540798976, + "inactive_split_bytes_peak": 7690537472, + "inactive_split_bytes_peak_delta": 2459424256, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132097507328, + "reserved_bytes_current_delta": 27676114944, + "reserved_bytes_peak": 132097507328, + "reserved_bytes_peak_delta": 27676114944, + "segment_count_current": 788, + "segment_count_current_delta": 198, + "segment_count_peak": 788, + "segment_count_peak_delta": 198, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T114017033967Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.154259Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T114017033967Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T114017033967Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..85e3d8f5d7058a659595f07910eddf5e6a1b9916 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T114017033967Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c007e4d38cfc9c4d6383006ae1a284df88c74ba742d0e64661ca19509d99f459 +size 5853858 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T114017865216Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T114017865216Z.json new file mode 100644 index 0000000000000000000000000000000000000000..735027e422f73e1e684a2db73db07d86bee439bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T114017865216Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888519168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477456384, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888519168, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66888519168, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 4248973824, + "inactive_split_bytes_peak": 7690537472, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132097507328, + "reserved_bytes_peak": 132097507328, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:17.864743Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888519168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888519168, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66888519168, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 4248973824, + "inactive_split_bytes_peak": 7690537472, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 132097507328, + "reserved_bytes_peak": 132097507328, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:17.173180Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4248973824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7690537472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132097507328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132097507328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T114017865216Z.pickle", + "timestamp_utc": "2026-08-01T11:40:17.872486Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T114017865216Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T114017865216Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9b236ce2fd0ddce9e55327680bcab554e749b3c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T114017865216Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b4e3f054252bdf5d6195ad27520e5eb788c8fd72a956e0f33e6e6f8932abdf1 +size 244914 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T114018566261Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T114018566261Z.json new file mode 100644 index 0000000000000000000000000000000000000000..97c9fbb2b9931ca40330719eb7ae8460d1fee961 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T114018566261Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888519168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477456384, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888519168, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66888519168, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 4246876672, + "inactive_split_bytes_peak": 7690537472, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132097507328, + "reserved_bytes_peak": 132097507328, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:18.565848Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888519168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888519168, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66888519168, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 4248973824, + "inactive_split_bytes_peak": 7690537472, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 132097507328, + "reserved_bytes_peak": 132097507328, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:17.873547Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4246876672, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7690537472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132097507328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132097507328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T114018566261Z.pickle", + "timestamp_utc": "2026-08-01T11:40:18.572925Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T114018566261Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T114018566261Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2cfeefc54fbf3f8cd85f783956756166da1fe7a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T114018566261Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adf7b90e622ba2f3e28ceb5f588aadff7ac1dc374490fd251db8c85faebb2cc3 +size 244943 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T114019267967Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T114019267967Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea954a1d5da9d4bfed7b03540affa044d436dcf3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T114019267967Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888519168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477456384, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888519168, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66888519168, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 4246876672, + "inactive_split_bytes_peak": 7690537472, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132097507328, + "reserved_bytes_peak": 132097507328, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:19.267567Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888519168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888519168, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66888519168, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 4246876672, + "inactive_split_bytes_peak": 7690537472, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 132097507328, + "reserved_bytes_peak": 132097507328, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:18.573866Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4246876672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7690537472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132097507328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132097507328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T114019267967Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.274259Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T114019267967Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T114019267967Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9555ec0711cba0e9261c08674e45a717dbfcbdba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T114019267967Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc6abb276a983bf96a9824c516c8c4ba99c387198da40ad4dd14b750e4950036 +size 244912 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T114019969057Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T114019969057Z.json new file mode 100644 index 0000000000000000000000000000000000000000..085a4f7fa90ca8561fa2f70c525c393ad42117f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T114019969057Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888519168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477456384, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888519168, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66888519168, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 4246876672, + "inactive_split_bytes_peak": 7690537472, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132097507328, + "reserved_bytes_peak": 132097507328, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:19.968651Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888519168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888519168, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66888519168, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 4246876672, + "inactive_split_bytes_peak": 7690537472, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 132097507328, + "reserved_bytes_peak": 132097507328, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:19.275179Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4246876672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7690537472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132097507328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132097507328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T114019969057Z.pickle", + "timestamp_utc": "2026-08-01T11:40:19.975553Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T114019969057Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T114019969057Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..16f393df23fbf1b8a7d09b0296449d18e85ccd92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T114019969057Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8a4ee444ae0840ca86b7bf334171715f914a62a4b68529699c0862967e338b8 +size 244943 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T114020672925Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T114020672925Z.json new file mode 100644 index 0000000000000000000000000000000000000000..252c7f81a735402ee211bae3c08f544f2ee68f24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T114020672925Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888519168, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121477456384, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888519168, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66888519168, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 4246876672, + "inactive_split_bytes_peak": 7690537472, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132097507328, + "reserved_bytes_peak": 132097507328, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:20.672521Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888519168, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326691840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888519168, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66888519168, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 4246876672, + "inactive_split_bytes_peak": 7690537472, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 132097507328, + "reserved_bytes_peak": 132097507328, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:19.976487Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4246876672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7690537472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132097507328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132097507328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 788, + "segment_count_current_delta": 0, + "segment_count_peak": 788, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T114020672925Z.pickle", + "timestamp_utc": "2026-08-01T11:40:20.679364Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T114020672925Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T114020672925Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b19d8af6720af8f66dbf0d524a0f18dc638aa2c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T114020672925Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb746401718353f8a4c76eee4971411feb1360c948ecfe3deb2abb707f1e3e2b +size 244943 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T114113364317Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T114113364317Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3c2fe568a449fbb4fbc3668a903328a6e495f0ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T114113364317Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854884864, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 121477456384, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854884864, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66854884864, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 2294508032, + "inactive_split_bytes_peak": 7690537472, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132120576000, + "reserved_bytes_peak": 132120576000, + "segment_count_current": 790, + "segment_count_peak": 790, + "timestamp_utc": "2026-08-01T11:41:13.363836Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854884864, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103326691840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888519168, + "active_bytes_peak": 121477456384, + "allocated_bytes_current": 66888519168, + "allocated_bytes_peak": 103326691840, + "device": 7, + "inactive_split_bytes_current": 4246876672, + "inactive_split_bytes_peak": 7690537472, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 132097507328, + "reserved_bytes_peak": 132097507328, + "segment_count_current": 788, + "segment_count_peak": 788, + "timestamp_utc": "2026-08-01T11:40:20.680298Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2294508032, + "inactive_split_bytes_current_delta": -1952368640, + "inactive_split_bytes_peak": 7690537472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132120576000, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132120576000, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 790, + "segment_count_current_delta": 2, + "segment_count_peak": 790, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T114113364317Z.pickle", + "timestamp_utc": "2026-08-01T11:41:13.453260Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T114113364317Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T114113364317Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4f499f19da714f0eb4668f9cb522efd23d40a702 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T114113364317Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:102fff74a19c09ef3429d9ea88f818dbea7153e76d49bed531cdcb4d65b8c7bb +size 4123471 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..2818d4063587de3a98cd725c89af94b8185a464e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 2048 +capacity: 1.00 +expert_dispatch_overlap: off +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6001 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 240 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T114410176213Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T114410176213Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db564143118e1e03ef510d431b1e4d7fcc4e7551 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T114410176213Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457436160, + "active_bytes_current_delta": 37225980928, + "active_bytes_peak": 79584741376, + "active_bytes_peak_delta": 44118405120, + "after": { + "active_bytes_current": 72457436160, + "active_bytes_peak": 79584741376, + "allocated_bytes_current": 72457436160, + "allocated_bytes_peak": 79584741376, + "device": 0, + "inactive_split_bytes_current": 4069737472, + "inactive_split_bytes_peak": 7353452032, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 90395639808, + "reserved_bytes_peak": 90395639808, + "segment_count_current": 444, + "segment_count_peak": 444, + "timestamp_utc": "2026-08-01T11:44:10.175625Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457436160, + "allocated_bytes_current_delta": 37225980928, + "allocated_bytes_peak": 79584741376, + "allocated_bytes_peak_delta": 44118405120, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T11:43:01.037047Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4069737472, + "inactive_split_bytes_current_delta": 3934821376, + "inactive_split_bytes_peak": 7353452032, + "inactive_split_bytes_peak_delta": 7192820224, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 90395639808, + "reserved_bytes_current_delta": 51149537280, + "reserved_bytes_peak": 90395639808, + "reserved_bytes_peak_delta": 51149537280, + "segment_count_current": 444, + "segment_count_current_delta": 372, + "segment_count_peak": 444, + "segment_count_peak_delta": 372, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T114410176213Z.pickle", + "timestamp_utc": "2026-08-01T11:44:10.293686Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T114410176213Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T114410176213Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..063987dc236150f5f81219ccd75ad4aff73bba6e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T114410176213Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8356ca8d7d1e7d022700e3cf3414c2e933d61b320195820cb43a775e547ae06d +size 5934640 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T114411239956Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T114411239956Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96a045d27ee99f6d18a8c148269466d4b605ceb3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T114411239956Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219342848, + "active_bytes_peak_delta": 23634601472, + "after": { + "active_bytes_current": 72457436160, + "active_bytes_peak": 103219342848, + "allocated_bytes_current": 72457436160, + "allocated_bytes_peak": 103219342848, + "device": 0, + "inactive_split_bytes_current": 4126360576, + "inactive_split_bytes_peak": 7353452032, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104884862976, + "reserved_bytes_peak": 104884862976, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T11:44:11.239553Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219342848, + "allocated_bytes_peak_delta": 23634601472, + "before": { + "active_bytes_current": 72457436160, + "active_bytes_peak": 79584741376, + "allocated_bytes_current": 72457436160, + "allocated_bytes_peak": 79584741376, + "device": 0, + "inactive_split_bytes_current": 4069737472, + "inactive_split_bytes_peak": 7353452032, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 90395639808, + "reserved_bytes_peak": 90395639808, + "segment_count_current": 444, + "segment_count_peak": 444, + "timestamp_utc": "2026-08-01T11:44:10.313395Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4126360576, + "inactive_split_bytes_current_delta": 56623104, + "inactive_split_bytes_peak": 7353452032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104884862976, + "reserved_bytes_current_delta": 14489223168, + "reserved_bytes_peak": 104884862976, + "reserved_bytes_peak_delta": 14489223168, + "segment_count_current": 538, + "segment_count_current_delta": 94, + "segment_count_peak": 538, + "segment_count_peak_delta": 94, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T114411239956Z.pickle", + "timestamp_utc": "2026-08-01T11:44:11.353417Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T114411239956Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T114411239956Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bcc809cb6ec6a5bbd616635048b419ed6af9e8d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T114411239956Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf61f0d3784639bb4c68ffab883209454ea02afc1389bdd3abaf3544d4d0537b +size 6317331 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T114412297495Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T114412297495Z.json new file mode 100644 index 0000000000000000000000000000000000000000..86a76bf0b60ba9a55dde4129e2379e097726697e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T114412297495Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103222874112, + "active_bytes_peak_delta": 3531264, + "after": { + "active_bytes_current": 72457436160, + "active_bytes_peak": 103222874112, + "allocated_bytes_current": 72457436160, + "allocated_bytes_peak": 103222874112, + "device": 0, + "inactive_split_bytes_current": 4484973568, + "inactive_split_bytes_peak": 7353452032, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104884862976, + "reserved_bytes_peak": 104884862976, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T11:44:12.297096Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103222874112, + "allocated_bytes_peak_delta": 3531264, + "before": { + "active_bytes_current": 72457436160, + "active_bytes_peak": 103219342848, + "allocated_bytes_current": 72457436160, + "allocated_bytes_peak": 103219342848, + "device": 0, + "inactive_split_bytes_current": 4126360576, + "inactive_split_bytes_peak": 7353452032, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104884862976, + "reserved_bytes_peak": 104884862976, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T11:44:11.369432Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4484973568, + "inactive_split_bytes_current_delta": 358612992, + "inactive_split_bytes_peak": 7353452032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104884862976, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104884862976, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T114412297495Z.pickle", + "timestamp_utc": "2026-08-01T11:44:12.408713Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T114412297495Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T114412297495Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ac3ec07588edec9e3db707f0ab82e4d4f268c641 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T114412297495Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:445af269efb086f616271179b40e5dee0575d6eec282d32e28138434a623bcc9 +size 6593103 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T114415009420Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T114415009420Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94233c275cf7cb668e2a33a2cf2577d529f6fb2a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T114415009420Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474479616, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121390186496, + "active_bytes_peak_delta": 18167312384, + "after": { + "active_bytes_current": 72474479616, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 72474479616, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_peak": 10150159872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_peak": 132357554176, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:44:15.008866Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474479616, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103235379200, + "allocated_bytes_peak_delta": 12505088, + "before": { + "active_bytes_current": 72457436160, + "active_bytes_peak": 103222874112, + "allocated_bytes_current": 72457436160, + "allocated_bytes_peak": 103222874112, + "device": 0, + "inactive_split_bytes_current": 4484973568, + "inactive_split_bytes_peak": 7353452032, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104884862976, + "reserved_bytes_peak": 104884862976, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T11:44:12.424355Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_current_delta": 721154048, + "inactive_split_bytes_peak": 10150159872, + "inactive_split_bytes_peak_delta": 2796707840, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_current_delta": 27472691200, + "reserved_bytes_peak": 132357554176, + "reserved_bytes_peak_delta": 27472691200, + "segment_count_current": 682, + "segment_count_current_delta": 144, + "segment_count_peak": 682, + "segment_count_peak_delta": 144, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T114415009420Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.132242Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T114415009420Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T114415009420Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..88b3d857a108d415f8e846a6870b5afec399ddf1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T114415009420Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ccece633cd4047869b0345711b72fdc3e78864bcf6f91a8afea7bbf45b79fae +size 6342096 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T114415985568Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T114415985568Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e491bf2d5887cef59a57cbca0942a9563ca6a770 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T114415985568Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474479616, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121390186496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474479616, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 72474479616, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_peak": 10150159872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_peak": 132357554176, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:44:15.985099Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474479616, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103235379200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474479616, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 72474479616, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_peak": 10150159872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_peak": 132357554176, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:44:15.149683Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10150159872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132357554176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 682, + "segment_count_current_delta": 0, + "segment_count_peak": 682, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T114415985568Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.991399Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T114415985568Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T114415985568Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c0f72878f31fa424b29a84f1dc7a25d5c41d8fbf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T114415985568Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c40f0771438c53ffc2eace14a870b594bc6a2deffc77ea3ff0755445056f9994 +size 186131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T114416815077Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T114416815077Z.json new file mode 100644 index 0000000000000000000000000000000000000000..38c2f53c1a0ab5635a81a0f55b83c42501f8412b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T114416815077Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474479616, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121390186496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474479616, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 72474479616, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_peak": 10150159872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_peak": 132357554176, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:44:16.814626Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474479616, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103235379200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474479616, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 72474479616, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_peak": 10150159872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_peak": 132357554176, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:44:15.992306Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10150159872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132357554176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 682, + "segment_count_current_delta": 0, + "segment_count_peak": 682, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T114416815077Z.pickle", + "timestamp_utc": "2026-08-01T11:44:16.820937Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T114416815077Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T114416815077Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3be5b0d29ec4b9a9f9254073d9e48326ed9a996e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T114416815077Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:662a5e23be887a8c3a7f4c23525041ee0a344fbceb11a4209e466eaba497c04a +size 186131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T114417642812Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T114417642812Z.json new file mode 100644 index 0000000000000000000000000000000000000000..923c273b4d1d8c7cc2977daed98e720cf0a6b2ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T114417642812Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474479616, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121390186496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474479616, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 72474479616, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_peak": 10150159872, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_peak": 132357554176, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:44:17.642397Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474479616, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103235379200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474479616, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 72474479616, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_peak": 10150159872, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_peak": 132357554176, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:44:16.821844Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10150159872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132357554176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 682, + "segment_count_current_delta": 0, + "segment_count_peak": 682, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T114417642812Z.pickle", + "timestamp_utc": "2026-08-01T11:44:17.647950Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T114417642812Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T114417642812Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de15d11ae5fc8fa9e4b43fbec6ae892be094d89b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T114417642812Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9400348b4752b116e132dc2e052a02fd3879d3f4130fa7f5bb05acf017d2d01b +size 186131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T114418470607Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T114418470607Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bee252594f0b3161aba64cb780d032f1513bfad4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T114418470607Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474479616, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121390186496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474479616, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 72474479616, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_peak": 10150159872, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_peak": 132357554176, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:44:18.470212Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474479616, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103235379200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474479616, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 72474479616, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_peak": 10150159872, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_peak": 132357554176, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:44:17.648847Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10150159872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132357554176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 682, + "segment_count_current_delta": 0, + "segment_count_peak": 682, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T114418470607Z.pickle", + "timestamp_utc": "2026-08-01T11:44:18.475684Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T114418470607Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T114418470607Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3b391f5628715db097810c1401100c12a3a7ce6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T114418470607Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8274081321d495a79e28948b333ed1ff2756319a1171197e7ac874f74d885d11 +size 186131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T114419299113Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T114419299113Z.json new file mode 100644 index 0000000000000000000000000000000000000000..49bdaa9ef391b7c88383c055136fc5043492a5b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T114419299113Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474479616, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121390186496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474479616, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 72474479616, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_peak": 10150159872, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_peak": 132357554176, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:44:19.298720Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474479616, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103235379200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474479616, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 72474479616, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_peak": 10150159872, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_peak": 132357554176, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:44:18.476562Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10150159872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132357554176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 682, + "segment_count_current_delta": 0, + "segment_count_peak": 682, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T114419299113Z.pickle", + "timestamp_utc": "2026-08-01T11:44:19.307482Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T114419299113Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T114419299113Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..194e0113ba083eda274957b06c6c0dfd80e20335 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T114419299113Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7315f1600bb275bdaac91c6aa8512c2d96656f487243e74315d06aad300cb5e1 +size 186131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T114517752489Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T114517752489Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5ccc9c71bff3257e0ca5968768f672b45ceb302 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T114517752489Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61217199104, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 121390186496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61217199104, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 61217199104, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 4155223040, + "inactive_split_bytes_peak": 10150159872, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 132380622848, + "reserved_bytes_peak": 132380622848, + "segment_count_current": 684, + "segment_count_peak": 684, + "timestamp_utc": "2026-08-01T11:45:17.752016Z", + "world_size": 8 + }, + "allocated_bytes_current": 61217199104, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103235379200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474479616, + "active_bytes_peak": 121390186496, + "allocated_bytes_current": 72474479616, + "allocated_bytes_peak": 103235379200, + "device": 0, + "inactive_split_bytes_current": 5206127616, + "inactive_split_bytes_peak": 10150159872, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 132357554176, + "reserved_bytes_peak": 132357554176, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:44:19.308388Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4155223040, + "inactive_split_bytes_current_delta": -1050904576, + "inactive_split_bytes_peak": 10150159872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 132380622848, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132380622848, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 684, + "segment_count_current_delta": 2, + "segment_count_peak": 684, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T114517752489Z.pickle", + "timestamp_utc": "2026-08-01T11:45:17.839610Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T114517752489Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T114517752489Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..758c87174bb6dca466a7348f1962755410dc7293 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T114517752489Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ece753057a78f0b5bba46ef33601acf80fedaaac5c460d06374b365e319de06 +size 4107208 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T114410176184Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T114410176184Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cdd102a3fad1441080075c003c068214159b2ee7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T114410176184Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 79588091904, + "active_bytes_peak_delta": 44121755648, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 79588091904, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 79588091904, + "device": 1, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_peak": 7226424832, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 90857013248, + "reserved_bytes_peak": 90857013248, + "segment_count_current": 447, + "segment_count_peak": 447, + "timestamp_utc": "2026-08-01T11:44:10.175540Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 79588091904, + "allocated_bytes_peak_delta": 44121755648, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:43:01.037257Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_current_delta": 3675151360, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 7066038784, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 90857013248, + "reserved_bytes_current_delta": 51613007872, + "reserved_bytes_peak": 90857013248, + "reserved_bytes_peak_delta": 51613007872, + "segment_count_current": 447, + "segment_count_current_delta": 376, + "segment_count_peak": 447, + "segment_count_peak_delta": 376, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T114410176184Z.pickle", + "timestamp_utc": "2026-08-01T11:44:10.292861Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T114410176184Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T114410176184Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a7adccca0bf5c3820c74b3bd9ca6d6d1844dd33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T114410176184Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d8a26993f910751526b93f7af4acda986f3bc9f2e1f39900f8e244c86152ba4 +size 5926690 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T114411239956Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T114411239956Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9257478e296a0fb76ac6f0b31367c002f99539a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T114411239956Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103223705600, + "active_bytes_peak_delta": 23635613696, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103223705600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103223705600, + "device": 1, + "inactive_split_bytes_current": 4510516224, + "inactive_split_bytes_peak": 7226424832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:11.239558Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223705600, + "allocated_bytes_peak_delta": 23635613696, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 79588091904, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 79588091904, + "device": 1, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_peak": 7226424832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 90857013248, + "reserved_bytes_peak": 90857013248, + "segment_count_current": 447, + "segment_count_peak": 447, + "timestamp_utc": "2026-08-01T11:44:10.312353Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4510516224, + "inactive_split_bytes_current_delta": 702545920, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104433975296, + "reserved_bytes_current_delta": 13576962048, + "reserved_bytes_peak": 104433975296, + "reserved_bytes_peak_delta": 13576962048, + "segment_count_current": 532, + "segment_count_current_delta": 85, + "segment_count_peak": 532, + "segment_count_peak_delta": 85, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T114411239956Z.pickle", + "timestamp_utc": "2026-08-01T11:44:11.351983Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T114411239956Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T114411239956Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..282300c3bcaa7444b84b9288271cd91eb4a9c300 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T114411239956Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffacedf5388a518e9256d94bcc3fb6f21ee63e007f4ea799aee5cea8339e680e +size 5916521 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T114412297532Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T114412297532Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b90f8c7cf8460f6eb5b3fe13122d3b3d141e310e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T114412297532Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103232012288, + "active_bytes_peak_delta": 8306688, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103232012288, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 5041095680, + "inactive_split_bytes_peak": 7226424832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:12.297127Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 8306688, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103223705600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103223705600, + "device": 1, + "inactive_split_bytes_current": 4510516224, + "inactive_split_bytes_peak": 7226424832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:11.368517Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5041095680, + "inactive_split_bytes_current_delta": 530579456, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104433975296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104433975296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T114412297532Z.pickle", + "timestamp_utc": "2026-08-01T11:44:12.408629Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T114412297532Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T114412297532Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..62be320c4b9d205795621f12e3755d348f28894d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T114412297532Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75252a849e209fdd50ba60e9c45345ed0d1f80703767bcf0c7fa2958c7757f83 +size 5920264 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T114415009394Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T114415009394Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9b3e8cce04c98404262a7fde79e9544f1aa8e6d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T114415009394Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 18147966976, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:15.008864Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103232012288, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 5041095680, + "inactive_split_bytes_peak": 7226424832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:12.424691Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 1104932864, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 3938326016, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 28512878592, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 28512878592, + "segment_count_current": 678, + "segment_count_current_delta": 146, + "segment_count_peak": 678, + "segment_count_peak_delta": 146, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T114415009394Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.132239Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T114415009394Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T114415009394Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8fed6cbf422a0078d14c8505478b71e5aa882b23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T114415009394Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97cc665583270a650f98a8456bd4e81576e8f54e4e5fcdbf980dbd87c31e3cbd +size 6233945 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T114415985578Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T114415985578Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6be41bd22293818396779cec31afc7e906fec25 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T114415985578Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:15.985098Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:15.152586Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T114415985578Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.991137Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T114415985578Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T114415985578Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dd0c62986d9b4df407cf1b90b6b63261b1806b62 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T114415985578Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c7e242d1bf366524626f695911688571a0e9338c1cb0ae5fb6b48a1ff078031 +size 183161 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T114416815010Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T114416815010Z.json new file mode 100644 index 0000000000000000000000000000000000000000..83e2f1849b3bb955f3c1b08d2661abf8c2ed36a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T114416815010Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:16.814590Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:15.992034Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T114416815010Z.pickle", + "timestamp_utc": "2026-08-01T11:44:16.820276Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T114416815010Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T114416815010Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1b5b7f18bbe7dcebb49c2c47513ca22a2f6fded9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T114416815010Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:831ecbe50ce7c4a983bae9495eb8a6ed23ccc5b92f8cbccb14434e79a6000bc8 +size 183161 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T114417642830Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T114417642830Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d37b0bbc98c01ab889ca541790638f77736d4472 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T114417642830Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:17.642425Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:16.821179Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T114417642830Z.pickle", + "timestamp_utc": "2026-08-01T11:44:17.647800Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T114417642830Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T114417642830Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ce368f6e3082f72cff62f1fda4dff0e5ef6563c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T114417642830Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9e44dddbb7b5dfa8987e98ea0630cabd56063d37853b0842ee6d56c2482f81b +size 183161 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T114418470588Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T114418470588Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b6277522dfc799d9580626e79fe24294f0a08f5e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T114418470588Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:18.470210Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:17.648701Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T114418470588Z.pickle", + "timestamp_utc": "2026-08-01T11:44:18.475451Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T114418470588Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T114418470588Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da4c8596aa32ec8924287dd9262399d06cb48d6a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T114418470588Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a8217404be6132e7a7b4f2a9ee044e89f77e86d81a382d27b0bf744d6d63876 +size 183161 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T114419299099Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T114419299099Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e1d2086821ea2cd8707ff7b5dbeecf039b07243e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T114419299099Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:19.298717Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:18.476303Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T114419299099Z.pickle", + "timestamp_utc": "2026-08-01T11:44:19.303932Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T114419299099Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T114419299099Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5de538f5846b678146beda02b277f60b5ba0fc2b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T114419299099Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3de692171b6fa46bca9f42eab0fbd1152e0f1295843b6ba6f0089cd801fa882 +size 183161 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T114517752490Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T114517752490Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c8f330ba695252e542b11453d4e808c715a32fbe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T114517752490Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 5095123968, + "inactive_split_bytes_peak": 11164750848, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 132969922560, + "reserved_bytes_peak": 132969922560, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:45:17.752002Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 1, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:19.304801Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5095123968, + "inactive_split_bytes_current_delta": -1050904576, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 132969922560, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132969922560, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 680, + "segment_count_current_delta": 2, + "segment_count_peak": 680, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T114517752490Z.pickle", + "timestamp_utc": "2026-08-01T11:45:17.832118Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T114517752490Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T114517752490Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1e74419258262a9d1ed32f32d3af3c120e339fee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T114517752490Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72d8fe38ce414ad80b1be42035a74444e9d375b881be704d04449239f04a71df +size 3978196 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T114410176209Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T114410176209Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c4e5919b9b09b8ec22e90e1befd1391b4629fe2d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T114410176209Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 79588091904, + "active_bytes_peak_delta": 44121755648, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 79588091904, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 79588091904, + "device": 2, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_peak": 7226424832, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 90857013248, + "reserved_bytes_peak": 90857013248, + "segment_count_current": 447, + "segment_count_peak": 447, + "timestamp_utc": "2026-08-01T11:44:10.175558Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 79588091904, + "allocated_bytes_peak_delta": 44121755648, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:43:01.037287Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_current_delta": 3675151360, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 7066038784, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 90857013248, + "reserved_bytes_current_delta": 51613007872, + "reserved_bytes_peak": 90857013248, + "reserved_bytes_peak_delta": 51613007872, + "segment_count_current": 447, + "segment_count_current_delta": 376, + "segment_count_peak": 447, + "segment_count_peak_delta": 376, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T114410176209Z.pickle", + "timestamp_utc": "2026-08-01T11:44:10.294839Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T114410176209Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T114410176209Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f7ac9bf9d35abc15a5ca878a6e16fcf49181b3cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T114410176209Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f35d949d738dbdf4a65c1a859384fa4421e40b4ab0ff0a6316598e2f16478b0 +size 5926689 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T114411239994Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T114411239994Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3aa8342b4b553a44658e6694825c5791eae42515 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T114411239994Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103223705600, + "active_bytes_peak_delta": 23635613696, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103223705600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103223705600, + "device": 2, + "inactive_split_bytes_current": 4510516224, + "inactive_split_bytes_peak": 7226424832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:11.239578Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223705600, + "allocated_bytes_peak_delta": 23635613696, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 79588091904, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 79588091904, + "device": 2, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_peak": 7226424832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 90857013248, + "reserved_bytes_peak": 90857013248, + "segment_count_current": 447, + "segment_count_peak": 447, + "timestamp_utc": "2026-08-01T11:44:10.314077Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4510516224, + "inactive_split_bytes_current_delta": 702545920, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104433975296, + "reserved_bytes_current_delta": 13576962048, + "reserved_bytes_peak": 104433975296, + "reserved_bytes_peak_delta": 13576962048, + "segment_count_current": 532, + "segment_count_current_delta": 85, + "segment_count_peak": 532, + "segment_count_peak_delta": 85, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T114411239994Z.pickle", + "timestamp_utc": "2026-08-01T11:44:11.352278Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T114411239994Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T114411239994Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d5f99a6145c026a4109ee14af44be7ae7de5de0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T114411239994Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:185df4fd27854d377018190e53d2aefa2aac7af629e73c99c31bb17b8f638087 +size 5916519 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T114412297567Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T114412297567Z.json new file mode 100644 index 0000000000000000000000000000000000000000..740d684bea5776809738b8cca9c918fd5f0cd456 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T114412297567Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103232012288, + "active_bytes_peak_delta": 8306688, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103232012288, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 5043192832, + "inactive_split_bytes_peak": 7226424832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:12.297151Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 8306688, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103223705600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103223705600, + "device": 2, + "inactive_split_bytes_current": 4510516224, + "inactive_split_bytes_peak": 7226424832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:11.367975Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5043192832, + "inactive_split_bytes_current_delta": 532676608, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104433975296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104433975296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T114412297567Z.pickle", + "timestamp_utc": "2026-08-01T11:44:12.406887Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T114412297567Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T114412297567Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72972a0c1ffc04c5b82c02b243e3ebe73f65a5d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T114412297567Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:debd7c9a3a5f2e8a8a463f4f1845971ed73c30377ae996cda0dc6716abae6e3f +size 5920226 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T114415009571Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T114415009571Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7136367c23a7a81bfa1e302cfdd4edb6ea0ab3b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T114415009571Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 18147966976, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11166848000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:15.008963Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103232012288, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 5043192832, + "inactive_split_bytes_peak": 7226424832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:12.431558Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 1102835712, + "inactive_split_bytes_peak": 11166848000, + "inactive_split_bytes_peak_delta": 3940423168, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 28514975744, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 28514975744, + "segment_count_current": 679, + "segment_count_current_delta": 147, + "segment_count_peak": 679, + "segment_count_peak_delta": 147, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T114415009571Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.141817Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T114415009571Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T114415009571Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7305c583f59b0bfb6408a8cd16ccc6d4ee1ae579 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T114415009571Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ba44de06df16cbe8d0e434ff552e1ceae9da2b36559f9db73d2bc78cf61fff3 +size 6233959 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T114415985621Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T114415985621Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d169c51db6c426b2117bd10ec89659f86406a179 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T114415985621Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11166848000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:15.985116Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11166848000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:15.160458Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11166848000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T114415985621Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.991299Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T114415985621Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T114415985621Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..46f83066994d2cb412264237b1fffda8c4105be0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T114415985621Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcf073571493d12545d121b001c50be77fabba8f7436983b4147d21aa6a9cad4 +size 183227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T114416815088Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T114416815088Z.json new file mode 100644 index 0000000000000000000000000000000000000000..838ff5c51efaabdd49dac5d3131928670f56a31d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T114416815088Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11166848000, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:16.814629Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11166848000, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:15.992254Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11166848000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T114416815088Z.pickle", + "timestamp_utc": "2026-08-01T11:44:16.820802Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T114416815088Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T114416815088Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de8af0c31e33c2305f391b77d7b91fc16cba14e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T114416815088Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7ba94dd3ddc43a3d9f18a4b2e536d1ab10b83451517d3e9a8dcbd76687941fd +size 183227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T114417642849Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T114417642849Z.json new file mode 100644 index 0000000000000000000000000000000000000000..862f991ef12fd9a17cfbd179e47134a507a49c49 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T114417642849Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11166848000, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:17.642427Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11166848000, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:16.821717Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11166848000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T114417642849Z.pickle", + "timestamp_utc": "2026-08-01T11:44:17.648106Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T114417642849Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T114417642849Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c9fc1569605d41f8a8ad80cf1484dc0c24c47c88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T114417642849Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:181e8e9a8eee74bf0a8e6cb41717768c364486dc468cb8d0cfc2304f14e4b8da +size 183227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T114418470635Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T114418470635Z.json new file mode 100644 index 0000000000000000000000000000000000000000..abbee1023dc482c1e7b5adac348a52891584f056 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T114418470635Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11166848000, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:18.470230Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11166848000, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:17.648993Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11166848000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T114418470635Z.pickle", + "timestamp_utc": "2026-08-01T11:44:18.475586Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T114418470635Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T114418470635Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..217f3a26059d8c84fc398d4535f9720cc14c05e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T114418470635Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:786a1eb47b0e51dafb6777c30ab652d19b86d53b4a41e610570a944a6c26f4bf +size 183227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T114419299117Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T114419299117Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e571fa454e43aa0c4fa47312ba9c958026c8f187 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T114419299117Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11166848000, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:19.298726Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11166848000, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:18.476465Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11166848000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T114419299117Z.pickle", + "timestamp_utc": "2026-08-01T11:44:19.303946Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T114419299117Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T114419299117Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a0fa121e99bf7fa3b954fa3aaf2ab0c8d5c3943f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T114419299117Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b86595c827d83eb7836e6b57b0c0950414c8c7f6122216c48a5bd1ed19ee24c8 +size 183227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T114517752492Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T114517752492Z.json new file mode 100644 index 0000000000000000000000000000000000000000..649bf8074c00573434366731bc9b805a7250d854 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T114517752492Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 5095123968, + "inactive_split_bytes_peak": 11166848000, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 132972019712, + "reserved_bytes_peak": 132972019712, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T11:45:17.751999Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 2, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11166848000, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:19.304854Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5095123968, + "inactive_split_bytes_current_delta": -1050904576, + "inactive_split_bytes_peak": 11166848000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 132972019712, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132972019712, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 681, + "segment_count_current_delta": 2, + "segment_count_peak": 681, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T114517752492Z.pickle", + "timestamp_utc": "2026-08-01T11:45:17.838032Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T114517752492Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T114517752492Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fd1855c9b4c441d3918bfa5c77deeb8a9991151a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T114517752492Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c233d8f0edfb748507e9c2c4101f628cc14026a55d30ce31ac437640a3e4810 +size 3978325 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T114410176238Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T114410176238Z.json new file mode 100644 index 0000000000000000000000000000000000000000..17f33adc94cc3150b76c3be43d7826fa557162a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T114410176238Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 79588091904, + "active_bytes_peak_delta": 44121755648, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 79588091904, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 79588091904, + "device": 3, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_peak": 7226424832, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 90857013248, + "reserved_bytes_peak": 90857013248, + "segment_count_current": 447, + "segment_count_peak": 447, + "timestamp_utc": "2026-08-01T11:44:10.175595Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 79588091904, + "allocated_bytes_peak_delta": 44121755648, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:43:01.037189Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_current_delta": 3675151360, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 7066038784, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 90857013248, + "reserved_bytes_current_delta": 51613007872, + "reserved_bytes_peak": 90857013248, + "reserved_bytes_peak_delta": 51613007872, + "segment_count_current": 447, + "segment_count_current_delta": 376, + "segment_count_peak": 447, + "segment_count_peak_delta": 376, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T114410176238Z.pickle", + "timestamp_utc": "2026-08-01T11:44:10.294170Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T114410176238Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T114410176238Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..559b3bff0020f64d5554616dd1f162f1a672b27b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T114410176238Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4f178170c450ea35cd91f4841d5cd92871d89dd0e9ae62a836498bd9affb315 +size 5926688 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T114411239985Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T114411239985Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1701621b3260db6d0f7283978036262f629f7180 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T114411239985Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103223705600, + "active_bytes_peak_delta": 23635613696, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103223705600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103223705600, + "device": 3, + "inactive_split_bytes_current": 4512613376, + "inactive_split_bytes_peak": 7226424832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:11.239577Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223705600, + "allocated_bytes_peak_delta": 23635613696, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 79588091904, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 79588091904, + "device": 3, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_peak": 7226424832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 90857013248, + "reserved_bytes_peak": 90857013248, + "segment_count_current": 447, + "segment_count_peak": 447, + "timestamp_utc": "2026-08-01T11:44:10.313450Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4512613376, + "inactive_split_bytes_current_delta": 704643072, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104433975296, + "reserved_bytes_current_delta": 13576962048, + "reserved_bytes_peak": 104433975296, + "reserved_bytes_peak_delta": 13576962048, + "segment_count_current": 532, + "segment_count_current_delta": 85, + "segment_count_peak": 532, + "segment_count_peak_delta": 85, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T114411239985Z.pickle", + "timestamp_utc": "2026-08-01T11:44:11.352428Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T114411239985Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T114411239985Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe763a257f0c7437461722d356606b59eb8bf528 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T114411239985Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4ea71d009611b52c7f64f613df83862c43ccbdcd33793a8457f4b372f5b0334 +size 5916514 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T114412297539Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T114412297539Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b6fdaf74d110df3ae1d135cdb55372185beccdc6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T114412297539Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103232012288, + "active_bytes_peak_delta": 8306688, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103232012288, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 5043192832, + "inactive_split_bytes_peak": 7226424832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:12.297123Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 8306688, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103223705600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103223705600, + "device": 3, + "inactive_split_bytes_current": 4512613376, + "inactive_split_bytes_peak": 7226424832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:11.368593Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5043192832, + "inactive_split_bytes_current_delta": 530579456, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104433975296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104433975296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T114412297539Z.pickle", + "timestamp_utc": "2026-08-01T11:44:12.410870Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T114412297539Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T114412297539Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..986ec8d6767f07ca4a5dc58e10f1b2b733db54e7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T114412297539Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35d454784da91ad0c68ca480516649eef6dc6c3b1f35287d8aa867329eed0768 +size 5920270 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T114415009489Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T114415009489Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80d9ef95b9c4156b994a85aa1f312d67ac03e840 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T114415009489Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 18147966976, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:15.008912Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103232012288, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 5043192832, + "inactive_split_bytes_peak": 7226424832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:12.433069Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 1102835712, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 3938326016, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 28514975744, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 28514975744, + "segment_count_current": 679, + "segment_count_current_delta": 147, + "segment_count_peak": 679, + "segment_count_peak_delta": 147, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T114415009489Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.134672Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T114415009489Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T114415009489Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a0eff350fe1ce602a9c3f0678a277113d766453a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T114415009489Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2db5b4bceace681b8ab6f57e68b8500ddc51d389b6e83593639879b750c70f5 +size 6233987 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T114415985564Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T114415985564Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a12d427c53302cd332b90f502700b35c7108fc52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T114415985564Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:15.985103Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:15.151831Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T114415985564Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.991277Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T114415985564Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T114415985564Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a0df011d08d052c86b60b529650598c145112b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T114415985564Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c13f90cb3c7cce51378287153a739ffbe4eb920667172b5afd223eaeb63d7d52 +size 183255 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T114416815073Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T114416815073Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad9128d3a87acaebe46bc7d9dc119b3aed75a012 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T114416815073Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:16.814637Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:15.992211Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T114416815073Z.pickle", + "timestamp_utc": "2026-08-01T11:44:16.820517Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T114416815073Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T114416815073Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5111fec4ba705b3c0823e84330c548faf0ed760 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T114416815073Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:325b7129949be0fe056c78a3ca1efd6581b2d44c41cc8c4be84e371f4804fb03 +size 183255 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T114417642839Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T114417642839Z.json new file mode 100644 index 0000000000000000000000000000000000000000..74b7a794398ae266bb0726eb4c061ee78e923778 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T114417642839Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:17.642434Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:16.821399Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T114417642839Z.pickle", + "timestamp_utc": "2026-08-01T11:44:17.647930Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T114417642839Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T114417642839Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b419629a0fc53f22dae3f27a03905e32310c9fca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T114417642839Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f152a53d474a023fd37dc5d0d1806bb0b3bb835f1d3db57200c467bfffa6a735 +size 183255 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T114418470595Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T114418470595Z.json new file mode 100644 index 0000000000000000000000000000000000000000..178b4efd0a6ee90854d7128396eb2ce4599a547b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T114418470595Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:18.470197Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:17.648784Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T114418470595Z.pickle", + "timestamp_utc": "2026-08-01T11:44:18.475441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T114418470595Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T114418470595Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7201b76756a520d538a3756ecc10d4e58ca70947 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T114418470595Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b90d1a25b596ebd6e34226b8118f2e08f34707a54596cdf544587abd19c18512 +size 183255 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T114419299266Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T114419299266Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae3ecd672315b140abafe774a2d7291c9368e43c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T114419299266Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:19.298855Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:18.476396Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T114419299266Z.pickle", + "timestamp_utc": "2026-08-01T11:44:19.305050Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T114419299266Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T114419299266Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6fa1aa53affed94dd451d09fae5a8755e724f51c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T114419299266Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c2c6d2ffdac88d4cab527cde8b07c28345fdfbf63dfec4dcc5d559c0078691f +size 183255 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T114517752488Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T114517752488Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db20fba88a726c261de993f7e697363b0c839753 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T114517752488Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 5095123968, + "inactive_split_bytes_peak": 11164750848, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 132972019712, + "reserved_bytes_peak": 132972019712, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T11:45:17.752004Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 3, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:19.305907Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5095123968, + "inactive_split_bytes_current_delta": -1050904576, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 132972019712, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132972019712, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 681, + "segment_count_current_delta": 2, + "segment_count_peak": 681, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T114517752488Z.pickle", + "timestamp_utc": "2026-08-01T11:45:17.836608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T114517752488Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T114517752488Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d16e525aad4f0c4809dc96ecc711f11a73862f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T114517752488Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3b168df97909b77c3c4cb67fbbd67466e83429a4e02e83e3e0f0f9b039b61a7 +size 3978289 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T114410176118Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T114410176118Z.json new file mode 100644 index 0000000000000000000000000000000000000000..245d45a473c7a3a7910960ac3ef613056689d84e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T114410176118Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459302912, + "active_bytes_current_delta": 37227405312, + "active_bytes_peak": 79586780160, + "active_bytes_peak_delta": 44120001536, + "after": { + "active_bytes_current": 72459302912, + "active_bytes_peak": 79586780160, + "allocated_bytes_current": 72459302912, + "allocated_bytes_peak": 79586780160, + "device": 4, + "inactive_split_bytes_current": 3707160576, + "inactive_split_bytes_peak": 7109459456, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 90622132224, + "reserved_bytes_peak": 90622132224, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T11:44:10.175514Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459302912, + "allocated_bytes_current_delta": 37227405312, + "allocated_bytes_peak": 79586780160, + "allocated_bytes_peak_delta": 44120001536, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:43:01.036849Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3707160576, + "inactive_split_bytes_current_delta": 3574784000, + "inactive_split_bytes_peak": 7109459456, + "inactive_split_bytes_peak_delta": 6956049920, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 90622132224, + "reserved_bytes_current_delta": 51378126848, + "reserved_bytes_peak": 90622132224, + "reserved_bytes_peak_delta": 51378126848, + "segment_count_current": 448, + "segment_count_current_delta": 377, + "segment_count_peak": 448, + "segment_count_peak_delta": 377, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T114410176118Z.pickle", + "timestamp_utc": "2026-08-01T11:44:10.294095Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T114410176118Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T114410176118Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d537d7d3707a5ed3775d0108f8a8567998cc35ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T114410176118Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31e49472c9706bc6335387e85470a27f43bfc85ebab1a92dd64a41b361bdd0b2 +size 5935091 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T114411239924Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T114411239924Z.json new file mode 100644 index 0000000000000000000000000000000000000000..958cc0b5404285c31a48305f3d408d3dbeea4f80 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T114411239924Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459302912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103230935040, + "active_bytes_peak_delta": 23644154880, + "after": { + "active_bytes_current": 72459302912, + "active_bytes_peak": 103230935040, + "allocated_bytes_current": 72459302912, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 4648781824, + "inactive_split_bytes_peak": 7109459456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104469626880, + "reserved_bytes_peak": 104469626880, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T11:44:11.239525Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459302912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103230935040, + "allocated_bytes_peak_delta": 23644154880, + "before": { + "active_bytes_current": 72459302912, + "active_bytes_peak": 79586780160, + "allocated_bytes_current": 72459302912, + "allocated_bytes_peak": 79586780160, + "device": 4, + "inactive_split_bytes_current": 3707160576, + "inactive_split_bytes_peak": 7109459456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 90622132224, + "reserved_bytes_peak": 90622132224, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T11:44:10.313404Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4648781824, + "inactive_split_bytes_current_delta": 941621248, + "inactive_split_bytes_peak": 7109459456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104469626880, + "reserved_bytes_current_delta": 13847494656, + "reserved_bytes_peak": 104469626880, + "reserved_bytes_peak_delta": 13847494656, + "segment_count_current": 536, + "segment_count_current_delta": 88, + "segment_count_peak": 536, + "segment_count_peak_delta": 88, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T114411239924Z.pickle", + "timestamp_utc": "2026-08-01T11:44:11.352317Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T114411239924Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T114411239924Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b3b898e54951afb1f97380a00b3b5eb6e7e05b28 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T114411239924Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9c03d284c722955909b52ba79393f18df6819ef5d9e781ed1c52f97e8df6b15 +size 5923794 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T114412297472Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T114412297472Z.json new file mode 100644 index 0000000000000000000000000000000000000000..10fe379ea7e2a682e28804d4937060ef52f8d533 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T114412297472Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459302912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103230935040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72459302912, + "active_bytes_peak": 103230935040, + "allocated_bytes_current": 72459302912, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 4237740032, + "inactive_split_bytes_peak": 7109459456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104469626880, + "reserved_bytes_peak": 104469626880, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T11:44:12.297071Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459302912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103230935040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72459302912, + "active_bytes_peak": 103230935040, + "allocated_bytes_current": 72459302912, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 4648781824, + "inactive_split_bytes_peak": 7109459456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104469626880, + "reserved_bytes_peak": 104469626880, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T11:44:11.368229Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4237740032, + "inactive_split_bytes_current_delta": -411041792, + "inactive_split_bytes_peak": 7109459456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104469626880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104469626880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T114412297472Z.pickle", + "timestamp_utc": "2026-08-01T11:44:12.406695Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T114412297472Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T114412297472Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..07b2f6c6675cfafce08e50a479f6dabd3272bc71 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T114412297472Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f52bd137072443bfcd278f3b566e4e934d41206ed5f739145f3ed60f6a7b397 +size 6595488 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T114415009452Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T114415009452Z.json new file mode 100644 index 0000000000000000000000000000000000000000..77b17a94175454a574ddab080d76dc817313b910 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T114415009452Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72476346368, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121384975360, + "active_bytes_peak_delta": 18154040320, + "after": { + "active_bytes_current": 72476346368, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 72476346368, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_peak": 10432195072, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132053467136, + "reserved_bytes_peak": 132053467136, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:44:15.008901Z", + "world_size": 8 + }, + "allocated_bytes_current": 72476346368, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103230935040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72459302912, + "active_bytes_peak": 103230935040, + "allocated_bytes_current": 72459302912, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 4237740032, + "inactive_split_bytes_peak": 7109459456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104469626880, + "reserved_bytes_peak": 104469626880, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T11:44:12.421779Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_current_delta": 1337716736, + "inactive_split_bytes_peak": 10432195072, + "inactive_split_bytes_peak_delta": 3322735616, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132053467136, + "reserved_bytes_current_delta": 27583840256, + "reserved_bytes_peak": 132053467136, + "reserved_bytes_peak_delta": 27583840256, + "segment_count_current": 680, + "segment_count_current_delta": 144, + "segment_count_peak": 680, + "segment_count_peak_delta": 144, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T114415009452Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.143012Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T114415009452Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T114415009452Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a6099e7bcaebb61611bd302b490aa9bac87979ef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T114415009452Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b2c8edc1ab4bc939d36aab7b4ddde83c604f02499ce05676d0c58c3ebb1e078 +size 6340944 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T114415985553Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T114415985553Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cb1fd82e8d73ac26d778eccc1c5cc20acb485e96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T114415985553Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72476346368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121384975360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72476346368, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 72476346368, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_peak": 10432195072, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132053467136, + "reserved_bytes_peak": 132053467136, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:44:15.985075Z", + "world_size": 8 + }, + "allocated_bytes_current": 72476346368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103230935040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72476346368, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 72476346368, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_peak": 10432195072, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 132053467136, + "reserved_bytes_peak": 132053467136, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:44:15.163681Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10432195072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132053467136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132053467136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 680, + "segment_count_current_delta": 0, + "segment_count_peak": 680, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T114415985553Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.991250Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T114415985553Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T114415985553Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ffda21fe9bbc1f37eac9f0cbceba1b968c713b36 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T114415985553Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cdd1826be582252bfc97f4772b65a6c7a003ac2bc5170eeba7b3a2bf63afc4f +size 186233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T114416815024Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T114416815024Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b17b0aaaf9fd7d1b23c2f2591e6136280a4116e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T114416815024Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72476346368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121384975360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72476346368, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 72476346368, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_peak": 10432195072, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132053467136, + "reserved_bytes_peak": 132053467136, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:44:16.814574Z", + "world_size": 8 + }, + "allocated_bytes_current": 72476346368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103230935040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72476346368, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 72476346368, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_peak": 10432195072, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 132053467136, + "reserved_bytes_peak": 132053467136, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:44:15.992141Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10432195072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132053467136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132053467136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 680, + "segment_count_current_delta": 0, + "segment_count_peak": 680, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T114416815024Z.pickle", + "timestamp_utc": "2026-08-01T11:44:16.820663Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T114416815024Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T114416815024Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..426f6681171ca8250bd2f53ecd5b9bb8a73ba166 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T114416815024Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:166b559c16f3fad0bd079df886136e17a3734601536a95333ad4cfde67d765c3 +size 186233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T114417642795Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T114417642795Z.json new file mode 100644 index 0000000000000000000000000000000000000000..28908a088516e41f87de2cba425336ac9f13f2bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T114417642795Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72476346368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121384975360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72476346368, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 72476346368, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_peak": 10432195072, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132053467136, + "reserved_bytes_peak": 132053467136, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:44:17.642388Z", + "world_size": 8 + }, + "allocated_bytes_current": 72476346368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103230935040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72476346368, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 72476346368, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_peak": 10432195072, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 132053467136, + "reserved_bytes_peak": 132053467136, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:44:16.821567Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10432195072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132053467136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132053467136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 680, + "segment_count_current_delta": 0, + "segment_count_peak": 680, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T114417642795Z.pickle", + "timestamp_utc": "2026-08-01T11:44:17.647976Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T114417642795Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T114417642795Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..be095bcd229d20c9e8770ee46d26b0561ae84116 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T114417642795Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3de70ea7dd09ead22c169b08028f960e36e8a68592f24d1e662f5ee747f8d161 +size 186233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T114418470568Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T114418470568Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb6218415a702d9d808e899f169713fab45dae12 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T114418470568Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72476346368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121384975360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72476346368, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 72476346368, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_peak": 10432195072, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132053467136, + "reserved_bytes_peak": 132053467136, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:44:18.470166Z", + "world_size": 8 + }, + "allocated_bytes_current": 72476346368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103230935040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72476346368, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 72476346368, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_peak": 10432195072, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 132053467136, + "reserved_bytes_peak": 132053467136, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:44:17.648882Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10432195072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132053467136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132053467136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 680, + "segment_count_current_delta": 0, + "segment_count_peak": 680, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T114418470568Z.pickle", + "timestamp_utc": "2026-08-01T11:44:18.475524Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T114418470568Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T114418470568Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c103af9c18e14c5bea6d3639607aaf393d3356e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T114418470568Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2adcb5e30019f3be27dd2cec505e5605e1af2f68b57ec5a4e067c60d4359212 +size 186233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T114419299054Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T114419299054Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2715a947429be5f4b295102675232e64b67d56d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T114419299054Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72476346368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121384975360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72476346368, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 72476346368, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_peak": 10432195072, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132053467136, + "reserved_bytes_peak": 132053467136, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:44:19.298669Z", + "world_size": 8 + }, + "allocated_bytes_current": 72476346368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103230935040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72476346368, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 72476346368, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_peak": 10432195072, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 132053467136, + "reserved_bytes_peak": 132053467136, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:44:18.476384Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10432195072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132053467136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132053467136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 680, + "segment_count_current_delta": 0, + "segment_count_peak": 680, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T114419299054Z.pickle", + "timestamp_utc": "2026-08-01T11:44:19.303918Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T114419299054Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T114419299054Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..281531a770430de4385f3ae4796dac3cfd0781f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T114419299054Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ec49d0d066371c7d4d7af3e70275f153cc93dbd122d3005fa0e720a136a1a6f +size 186233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T114517752495Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T114517752495Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2cbf24209653ca9ae725eda4b1516862cb39fc3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T114517752495Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61219065856, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 121384975360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61219065856, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 61219065856, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 4641992704, + "inactive_split_bytes_peak": 10432195072, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 132076535808, + "reserved_bytes_peak": 132076535808, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T11:45:17.752008Z", + "world_size": 8 + }, + "allocated_bytes_current": 61219065856, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103230935040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72476346368, + "active_bytes_peak": 121384975360, + "allocated_bytes_current": 72476346368, + "allocated_bytes_peak": 103230935040, + "device": 4, + "inactive_split_bytes_current": 5575456768, + "inactive_split_bytes_peak": 10432195072, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 132053467136, + "reserved_bytes_peak": 132053467136, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:44:19.304789Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4641992704, + "inactive_split_bytes_current_delta": -933464064, + "inactive_split_bytes_peak": 10432195072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 132076535808, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132076535808, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 682, + "segment_count_current_delta": 2, + "segment_count_peak": 682, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T114517752495Z.pickle", + "timestamp_utc": "2026-08-01T11:45:17.834237Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T114517752495Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T114517752495Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..080e7ba16afa8d5067b00679c62605d9d1c231e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T114517752495Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43679eebe25a9b7ad43d115b72e226806948d8eaae56903aca66fdafd8a53493 +size 4107356 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T114410176360Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T114410176360Z.json new file mode 100644 index 0000000000000000000000000000000000000000..beb875db131fc86cda9e62d840eebe597706f9af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T114410176360Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 79588091904, + "active_bytes_peak_delta": 44121755648, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 79588091904, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 79588091904, + "device": 5, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_peak": 7226424832, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 90857013248, + "reserved_bytes_peak": 90857013248, + "segment_count_current": 447, + "segment_count_peak": 447, + "timestamp_utc": "2026-08-01T11:44:10.175651Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 79588091904, + "allocated_bytes_peak_delta": 44121755648, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:43:01.037149Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_current_delta": 3675151360, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 7066038784, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 90857013248, + "reserved_bytes_current_delta": 51613007872, + "reserved_bytes_peak": 90857013248, + "reserved_bytes_peak_delta": 51613007872, + "segment_count_current": 447, + "segment_count_current_delta": 376, + "segment_count_peak": 447, + "segment_count_peak_delta": 376, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T114410176360Z.pickle", + "timestamp_utc": "2026-08-01T11:44:10.297787Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T114410176360Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T114410176360Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a905292ae1e0f1dd2e69226baf9685a09675e54 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T114410176360Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea09c618309ec62a5c126abe07d5714fda0e05508f6c18bbc7036d051a42b20d +size 5926664 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T114411239947Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T114411239947Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c36588af5b5b21ff69943916cc4324f2a533d7ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T114411239947Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103223705600, + "active_bytes_peak_delta": 23635613696, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103223705600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103223705600, + "device": 5, + "inactive_split_bytes_current": 4510516224, + "inactive_split_bytes_peak": 7226424832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:11.239555Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223705600, + "allocated_bytes_peak_delta": 23635613696, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 79588091904, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 79588091904, + "device": 5, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_peak": 7226424832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 90857013248, + "reserved_bytes_peak": 90857013248, + "segment_count_current": 447, + "segment_count_peak": 447, + "timestamp_utc": "2026-08-01T11:44:10.313654Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4510516224, + "inactive_split_bytes_current_delta": 702545920, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104433975296, + "reserved_bytes_current_delta": 13576962048, + "reserved_bytes_peak": 104433975296, + "reserved_bytes_peak_delta": 13576962048, + "segment_count_current": 532, + "segment_count_current_delta": 85, + "segment_count_peak": 532, + "segment_count_peak_delta": 85, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T114411239947Z.pickle", + "timestamp_utc": "2026-08-01T11:44:11.342422Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T114411239947Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T114411239947Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ae582cc7a6a59234c9f5ce28782af968e1d44ae7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T114411239947Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32f65db848631d9ee5e8eb27018b1fd51b4992f50d06d3a9f88aab6174b870fa +size 5916515 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T114412297524Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T114412297524Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d860adf93daec76f4271d8d98e8517c543a5932 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T114412297524Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103232012288, + "active_bytes_peak_delta": 8306688, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103232012288, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 5043192832, + "inactive_split_bytes_peak": 7226424832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:12.297113Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 8306688, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103223705600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103223705600, + "device": 5, + "inactive_split_bytes_current": 4510516224, + "inactive_split_bytes_peak": 7226424832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:11.353997Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5043192832, + "inactive_split_bytes_current_delta": 532676608, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104433975296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104433975296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T114412297524Z.pickle", + "timestamp_utc": "2026-08-01T11:44:12.400553Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T114412297524Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T114412297524Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..26a8ad450b2925a0226108cd7f4ea2220fb66f92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T114412297524Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a8ef8cebe46f3ff194849db9a03a96b4247612b8f5aa48c8406c7d3c8421c44 +size 5920267 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T114415009386Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T114415009386Z.json new file mode 100644 index 0000000000000000000000000000000000000000..17d050e8f2af3a7bf2c96a943457a1eda03b02bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T114415009386Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 18147966976, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:15.008855Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103232012288, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 5043192832, + "inactive_split_bytes_peak": 7226424832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:12.413916Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 1102835712, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 3938326016, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 28514975744, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 28514975744, + "segment_count_current": 679, + "segment_count_current_delta": 147, + "segment_count_peak": 679, + "segment_count_peak_delta": 147, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T114415009386Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.128381Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T114415009386Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T114415009386Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1231b06a1665b358795ba3f5ab0b1f8db3c151c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T114415009386Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e03e5840486cfdf2436e94c0c2c02da39fe7d9548399a3aa5b956bd1fe90cdc +size 6233959 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T114415985585Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T114415985585Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8a8c7b209b01605eddfabd5cca64b3f144897d35 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T114415985585Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:15.985093Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:15.142714Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T114415985585Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.991082Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T114415985585Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T114415985585Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a86e4265fb6551383e9072a133944618a384ed8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T114415985585Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc859ed64c17cd21c41e1df65a12819a8a8580b13947ec61f98b60037478418f +size 183227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T114416815175Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T114416815175Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ec7006c24f5059821bc600f0ff089696447c5fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T114416815175Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:16.814679Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:15.991973Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T114416815175Z.pickle", + "timestamp_utc": "2026-08-01T11:44:16.821360Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T114416815175Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T114416815175Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9fd554b1cbdff731f5583a890e35be7206a582d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T114416815175Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92c7989d4e1a89bc67ddda89e5ce568c7a63fcec058ea6cdb1df09b2db04b6b8 +size 183227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T114417642839Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T114417642839Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e75688b958fe172e6af778d99f70aaaed11fd1ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T114417642839Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:17.642418Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:16.822288Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T114417642839Z.pickle", + "timestamp_utc": "2026-08-01T11:44:17.648070Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T114417642839Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T114417642839Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..128ab602e870aefcef10863a31d1e29c8e755786 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T114417642839Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f986130d08197c49e9176c0c0540e2e89e159fa2795086ea56ec3ab4714ac13e +size 183227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T114418470648Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T114418470648Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3b69e8df9e229be47e632ee55b43e4f569feb47b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T114418470648Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:18.470232Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:17.648972Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T114418470648Z.pickle", + "timestamp_utc": "2026-08-01T11:44:18.475793Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T114418470648Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T114418470648Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6380f0003e161b8057eda0e2b01428faac29f257 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T114418470648Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:408219997d9fd72abc4c797495f3ba45d9a7071df288916520ee2f89cb63d526 +size 183227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T114419299112Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T114419299112Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eab4447c98bbac707daf170009a6c5ef643c20b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T114419299112Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:19.298711Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:18.476669Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132948951040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132948951040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T114419299112Z.pickle", + "timestamp_utc": "2026-08-01T11:44:19.304031Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T114419299112Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T114419299112Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e9a6aae6262785cd4b8c84d44a1183861e8d8ca1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T114419299112Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42a68be1aa791716866dbc257fd8f907c89848da52668d789ee750b16a1ff102 +size 183227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T114517752473Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T114517752473Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e69c73ff30f54d4a4a62d4f953540d11a10e6b1e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T114517752473Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 5095123968, + "inactive_split_bytes_peak": 11164750848, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 132972019712, + "reserved_bytes_peak": 132972019712, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T11:45:17.751994Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 5, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 132948951040, + "reserved_bytes_peak": 132948951040, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T11:44:19.304917Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5095123968, + "inactive_split_bytes_current_delta": -1050904576, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 132972019712, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132972019712, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 681, + "segment_count_current_delta": 2, + "segment_count_peak": 681, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T114517752473Z.pickle", + "timestamp_utc": "2026-08-01T11:45:17.834476Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T114517752473Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T114517752473Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e696bd1611abcf4c9c248aacfe01f1cf4a5308c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T114517752473Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e13772b358b81c99cac93014b9c205e1b28902c35d5d401d568a21bb568e6cbe +size 3978324 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T114410176200Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T114410176200Z.json new file mode 100644 index 0000000000000000000000000000000000000000..602f0b07193af4356e4d93506ec74232e41c3332 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T114410176200Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 79588091904, + "active_bytes_peak_delta": 44121755648, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 79588091904, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 79588091904, + "device": 6, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_peak": 7226424832, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 90857013248, + "reserved_bytes_peak": 90857013248, + "segment_count_current": 447, + "segment_count_peak": 447, + "timestamp_utc": "2026-08-01T11:44:10.175527Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 79588091904, + "allocated_bytes_peak_delta": 44121755648, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:43:01.037205Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_current_delta": 3675151360, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 7066038784, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 90857013248, + "reserved_bytes_current_delta": 51613007872, + "reserved_bytes_peak": 90857013248, + "reserved_bytes_peak_delta": 51613007872, + "segment_count_current": 447, + "segment_count_current_delta": 376, + "segment_count_peak": 447, + "segment_count_peak_delta": 376, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T114410176200Z.pickle", + "timestamp_utc": "2026-08-01T11:44:10.294480Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T114410176200Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T114410176200Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..25d693a0e1894d46b7b58867c26fea04c76105b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T114410176200Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:845e2de8a7a16bf88263004c074838b17647936473dd946a2f7fcd0424b88ee0 +size 5926694 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T114411240022Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T114411240022Z.json new file mode 100644 index 0000000000000000000000000000000000000000..adfdf554c4926607f4c992be881f700c903a6ccc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T114411240022Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103223705600, + "active_bytes_peak_delta": 23635613696, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103223705600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103223705600, + "device": 6, + "inactive_split_bytes_current": 4512613376, + "inactive_split_bytes_peak": 7226424832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:11.239617Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223705600, + "allocated_bytes_peak_delta": 23635613696, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 79588091904, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 79588091904, + "device": 6, + "inactive_split_bytes_current": 3807970304, + "inactive_split_bytes_peak": 7226424832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 90857013248, + "reserved_bytes_peak": 90857013248, + "segment_count_current": 447, + "segment_count_peak": 447, + "timestamp_utc": "2026-08-01T11:44:10.314959Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4512613376, + "inactive_split_bytes_current_delta": 704643072, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104433975296, + "reserved_bytes_current_delta": 13576962048, + "reserved_bytes_peak": 104433975296, + "reserved_bytes_peak_delta": 13576962048, + "segment_count_current": 532, + "segment_count_current_delta": 85, + "segment_count_peak": 532, + "segment_count_peak_delta": 85, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T114411240022Z.pickle", + "timestamp_utc": "2026-08-01T11:44:11.351970Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T114411240022Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T114411240022Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..016ae3d983b95df40da204092aae70a0dd10ad55 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T114411240022Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de96cc5957e2c6c231029ac81ede6dac2caad7a38497a4f2a1216556eb0a9d70 +size 5916518 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T114412297509Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T114412297509Z.json new file mode 100644 index 0000000000000000000000000000000000000000..54ed9de32ac74f114ac0839ee585ec7b334dba6a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T114412297509Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103232012288, + "active_bytes_peak_delta": 8306688, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103232012288, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 5041095680, + "inactive_split_bytes_peak": 7226424832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:12.297106Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 8306688, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103223705600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103223705600, + "device": 6, + "inactive_split_bytes_current": 4512613376, + "inactive_split_bytes_peak": 7226424832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:11.368152Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5041095680, + "inactive_split_bytes_current_delta": 528482304, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104433975296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104433975296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T114412297509Z.pickle", + "timestamp_utc": "2026-08-01T11:44:12.409240Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T114412297509Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T114412297509Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..55cce83b2a7e876eead63de5e11ab76f995e8188 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T114412297509Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00bc428c4e22b89bbc88c62526030d8fc33e9611b25cff8eadbdd399a451559f +size 5920266 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T114415009445Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T114415009445Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f40f08dfabb59c92af7f31524453db29aa5ae64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T114415009445Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 18147966976, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:15.008867Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 103232012288, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 5041095680, + "inactive_split_bytes_peak": 7226424832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:12.425544Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 1104932864, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 3938326016, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 28512878592, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 28512878592, + "segment_count_current": 678, + "segment_count_current_delta": 146, + "segment_count_peak": 678, + "segment_count_peak_delta": 146, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T114415009445Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.133514Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T114415009445Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T114415009445Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..405e5e39690b5ea0709a4c4e1a3e2e6528333590 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T114415009445Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd199a588f5fdae0fd0127208656678bf98595d0b32caec5f8495aa97c93cf13 +size 6233915 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T114415985554Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T114415985554Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d0c6fafdc598ee0ff5be7eb9014921a552772cc1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T114415985554Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:15.985062Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:15.150703Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T114415985554Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.991268Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T114415985554Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T114415985554Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..49d290cec4f2bfc5bfba778ceb35946b3df78112 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T114415985554Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8199d571d0ab04d57ca1972f9f98ef4d28c3ac9b3c7efbc0c22dbf112ec684c3 +size 183131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T114416815015Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T114416815015Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e91b4ba0899c2a0d55b3cd3f937249a9aeaff82 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T114416815015Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:16.814578Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:15.992184Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T114416815015Z.pickle", + "timestamp_utc": "2026-08-01T11:44:16.820406Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T114416815015Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T114416815015Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..46b361716d4778d40bd818f674902bc6c4cbb856 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T114416815015Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:947c2fa78577f2020d75d32416cfbee3ea9dfadd2475ae4f06a84109ef70e0cf +size 183131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T114417642938Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T114417642938Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1a342f21ad2b8fbd1acd5a6c5c61406b2e18e49 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T114417642938Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:17.642508Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:16.821280Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T114417642938Z.pickle", + "timestamp_utc": "2026-08-01T11:44:17.648688Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T114417642938Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T114417642938Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..462c6a54c7faad2d55ef7ab70afd2d15fd74e83e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T114417642938Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae7542837d2d27a7fafe0f2f7a70df27666630a80e761f62fcf550b37bf2563a +size 183131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T114418470770Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T114418470770Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87e2ad9001b91d304692e0b780e27bb54cf1b416 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T114418470770Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:18.470352Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:17.649564Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T114418470770Z.pickle", + "timestamp_utc": "2026-08-01T11:44:18.477409Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T114418470770Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T114418470770Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..15ecc731aaf5bdba8150ab76dbb990fab53ca570 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T114418470770Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66abf01f34e1c246e0282dc2b82f2e02e00339b4c36b7183855a754b4b8473e0 +size 183131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T114419299085Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T114419299085Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5fd88bf8a22f51741d202338cd8cee58d1f0bd42 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T114419299085Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:19.298699Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:18.478295Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T114419299085Z.pickle", + "timestamp_utc": "2026-08-01T11:44:19.304004Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T114419299085Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T114419299085Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c092c100f62b6a9c6cb000c4c33f4df5d30b1937 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T114419299085Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98a08803a8ac57ef537a4acf82b9b64b9836ba0db35670fd1e81e86db78e1c34 +size 183131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T114517752482Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T114517752482Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f88dbe2876dbc20236f3571099a6194d5db44fce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T114517752482Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 121379979264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 5095123968, + "inactive_split_bytes_peak": 11164750848, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 132969922560, + "reserved_bytes_peak": 132969922560, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:45:17.751997Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103232012288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 121379979264, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 103232012288, + "device": 6, + "inactive_split_bytes_current": 6146028544, + "inactive_split_bytes_peak": 11164750848, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:19.304870Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5095123968, + "inactive_split_bytes_current_delta": -1050904576, + "inactive_split_bytes_peak": 11164750848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 132969922560, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132969922560, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 680, + "segment_count_current_delta": 2, + "segment_count_peak": 680, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T114517752482Z.pickle", + "timestamp_utc": "2026-08-01T11:45:17.835372Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T114517752482Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T114517752482Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9588cc9eca015729cffe81affe6e9eb0d7f54368 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T114517752482Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f31b6965be207b254d2d02375248c54f9964bf8f7fca7976f699f8f0da2d1c7 +size 3978231 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T114410176124Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T114410176124Z.json new file mode 100644 index 0000000000000000000000000000000000000000..edaeee0fc9827c7a3387b0ff46012ab54ca77865 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T114410176124Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059840, + "active_bytes_current_delta": 37225604608, + "active_bytes_peak": 79588091904, + "active_bytes_peak_delta": 44121755648, + "after": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 79588091904, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 79588091904, + "device": 7, + "inactive_split_bytes_current": 3807969792, + "inactive_split_bytes_peak": 7226424832, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 90857013248, + "reserved_bytes_peak": 90857013248, + "segment_count_current": 447, + "segment_count_peak": 447, + "timestamp_utc": "2026-08-01T11:44:10.175523Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059840, + "allocated_bytes_current_delta": 37225604608, + "allocated_bytes_peak": 79588091904, + "allocated_bytes_peak_delta": 44121755648, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:43:01.037302Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3807969792, + "inactive_split_bytes_current_delta": 3675150848, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 7066038784, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 90857013248, + "reserved_bytes_current_delta": 51613007872, + "reserved_bytes_peak": 90857013248, + "reserved_bytes_peak_delta": 51613007872, + "segment_count_current": 447, + "segment_count_current_delta": 376, + "segment_count_peak": 447, + "segment_count_peak_delta": 376, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T114410176124Z.pickle", + "timestamp_utc": "2026-08-01T11:44:10.291570Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T114410176124Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T114410176124Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e384f655bc82a609d163825d3b499c74d7367d38 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T114410176124Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4ca6ba6408b463c8b6f28ce8cdf280b18e6ac6ee81a80e5181139a28bb2c369 +size 5928283 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T114411240008Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T114411240008Z.json new file mode 100644 index 0000000000000000000000000000000000000000..29c2e5b210d0524dd58590fcdc57eadd184c033f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T114411240008Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103223706112, + "active_bytes_peak_delta": 23635614208, + "after": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 103223706112, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 103223706112, + "device": 7, + "inactive_split_bytes_current": 4510515712, + "inactive_split_bytes_peak": 7226424832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:11.239618Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223706112, + "allocated_bytes_peak_delta": 23635614208, + "before": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 79588091904, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 79588091904, + "device": 7, + "inactive_split_bytes_current": 3807969792, + "inactive_split_bytes_peak": 7226424832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 90857013248, + "reserved_bytes_peak": 90857013248, + "segment_count_current": 447, + "segment_count_peak": 447, + "timestamp_utc": "2026-08-01T11:44:10.307390Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4510515712, + "inactive_split_bytes_current_delta": 702545920, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104433975296, + "reserved_bytes_current_delta": 13576962048, + "reserved_bytes_peak": 104433975296, + "reserved_bytes_peak_delta": 13576962048, + "segment_count_current": 532, + "segment_count_current_delta": 85, + "segment_count_peak": 532, + "segment_count_peak_delta": 85, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T114411240008Z.pickle", + "timestamp_utc": "2026-08-01T11:44:11.344389Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T114411240008Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T114411240008Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6efbffd06eccef99a612bfa2c8f3e1c6139af287 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T114411240008Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0daa33cf9ff9e95d3d8c14cf2bfbb5d72885d5011aedabcd4e11a3d16831952 +size 5918279 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T114412297589Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T114412297589Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e48c7a4cf44bf563827d9530e0d953c0145bacde --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T114412297589Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103232012800, + "active_bytes_peak_delta": 8306688, + "after": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 103232012800, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 5041095168, + "inactive_split_bytes_peak": 7226424832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:12.297192Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012800, + "allocated_bytes_peak_delta": 8306688, + "before": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 103223706112, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 103223706112, + "device": 7, + "inactive_split_bytes_current": 4510515712, + "inactive_split_bytes_peak": 7226424832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:11.356112Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5041095168, + "inactive_split_bytes_current_delta": 530579456, + "inactive_split_bytes_peak": 7226424832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104433975296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104433975296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T114412297589Z.pickle", + "timestamp_utc": "2026-08-01T11:44:12.400146Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T114412297589Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T114412297589Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6cdc8fe92e67e57e3892116ea4e192e8165da685 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T114412297589Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49420f57697c5af54c8f2f0d139524ef2d16c847981e6260a725c773c17cc7ea +size 5922028 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T114415009387Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T114415009387Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a33ca20bbd023e9e216fe66582b90b4958bcda61 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T114415009387Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 121379979776, + "active_bytes_peak_delta": 18147966976, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_peak": 11164750336, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:15.008888Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 103232012800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 103232012800, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 5041095168, + "inactive_split_bytes_peak": 7226424832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104433975296, + "reserved_bytes_peak": 104433975296, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T11:44:12.411893Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_current_delta": 1104932864, + "inactive_split_bytes_peak": 11164750336, + "inactive_split_bytes_peak_delta": 3938325504, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 28512878592, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 28512878592, + "segment_count_current": 678, + "segment_count_current_delta": 146, + "segment_count_peak": 678, + "segment_count_peak_delta": 146, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T114415009387Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.129072Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T114415009387Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T114415009387Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eafac59f8262bb452c7569762c2269115160ff8f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T114415009387Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cd4b2315c2eb48997a872716b6b67b04f682e10ff7b24f91e79f6236141653a +size 6235682 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T114415985713Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T114415985713Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e8d78cf2289d5e1837159a2dd838fbd586130c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T114415985713Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_peak": 11164750336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:15.985225Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_peak": 11164750336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:15.143459Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T114415985713Z.pickle", + "timestamp_utc": "2026-08-01T11:44:15.991673Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T114415985713Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T114415985713Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6b8b544821cbdd695474e0b5a776cec9d17326d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T114415985713Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bcebc5bc7699ca32c78a47c2bb6558666b1f73bf959e735bf052a16ccdb6567 +size 184929 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T114416815248Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T114416815248Z.json new file mode 100644 index 0000000000000000000000000000000000000000..953a86d764f1df9897bfd9ee3b9bf3f0be42ee75 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T114416815248Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_peak": 11164750336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:16.814737Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_peak": 11164750336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:15.992605Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T114416815248Z.pickle", + "timestamp_utc": "2026-08-01T11:44:16.821296Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T114416815248Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T114416815248Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e932be4fccda0a85f387503cfc78c2a78c1bfa8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T114416815248Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f38cb8ebf94bd213d00f301666f607585f6c00e79081d2f80244da99afb4b4b +size 184929 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T114417642947Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T114417642947Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc0d03bb86a9fd43797fa9e0edb6fd357eaa882c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T114417642947Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_peak": 11164750336, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:17.642504Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_peak": 11164750336, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:16.822194Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T114417642947Z.pickle", + "timestamp_utc": "2026-08-01T11:44:17.648013Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T114417642947Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T114417642947Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..77db2e97b2262f1e91a974f82ac7b95d3ca51dde --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T114417642947Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0364c4a8a5fa1999555089015f651ce6a0023af6599d042ea1eec92d84ebf8dc +size 184898 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T114418470660Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T114418470660Z.json new file mode 100644 index 0000000000000000000000000000000000000000..88e8b69a71fdd5059cb806bb4cd7b1fc6e200395 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T114418470660Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_peak": 11164750336, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:18.470252Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_peak": 11164750336, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:17.648937Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T114418470660Z.pickle", + "timestamp_utc": "2026-08-01T11:44:18.475549Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T114418470660Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T114418470660Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..52e48f5cf93247620e685e61fdad3534d9ede37e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T114418470660Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e22b44caa42224b922415a8cbcaf91d368b38ad9fd10fcfb4e0cd0ad34aa93c +size 184929 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T114419299144Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T114419299144Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7babb1a6ca973d587078abac4090c68a862790a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T114419299144Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 121379979776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_peak": 11164750336, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:19.298768Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232012800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_peak": 11164750336, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:18.476400Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11164750336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132946853888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 132946853888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 678, + "segment_count_current_delta": 0, + "segment_count_peak": 678, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T114419299144Z.pickle", + "timestamp_utc": "2026-08-01T11:44:19.303810Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T114419299144Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T114419299144Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76adf4e17dc96923916ef949cc04fa8cf95f3d27 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T114419299144Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:149231aeb83a2d9f65939539ab6672794aa2da54fee8b83f85f57d94b3923e8b +size 184929 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T114517752477Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T114517752477Z.json new file mode 100644 index 0000000000000000000000000000000000000000..30f75fae11494ab6a8493259d562cd40e81e5d48 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T114517752477Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822784, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 121379979776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822784, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 61216822784, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 5095123456, + "inactive_split_bytes_peak": 11164750336, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 132969922560, + "reserved_bytes_peak": 132969922560, + "segment_count_current": 680, + "segment_count_peak": 680, + "timestamp_utc": "2026-08-01T11:45:17.752025Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822784, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103232012800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 121379979776, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 103232012800, + "device": 7, + "inactive_split_bytes_current": 6146028032, + "inactive_split_bytes_peak": 11164750336, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 132946853888, + "reserved_bytes_peak": 132946853888, + "segment_count_current": 678, + "segment_count_peak": 678, + "timestamp_utc": "2026-08-01T11:44:19.304646Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5095123456, + "inactive_split_bytes_current_delta": -1050904576, + "inactive_split_bytes_peak": 11164750336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 132969922560, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 132969922560, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 680, + "segment_count_current_delta": 2, + "segment_count_peak": 680, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T114517752477Z.pickle", + "timestamp_utc": "2026-08-01T11:45:17.837059Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T114517752477Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T114517752477Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..86e6c92d72b962148edec214dbc827f70e1a4555 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T114517752477Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e333f80b1c4bf13ecf7ecfc4b3c05889360ba2048cb7a416f57887e4ee072bb2 +size 3979959 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..617f2b6d1b84b7a1d3cc77d9847c34c949790bac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 2048 +capacity: 1.00 +expert_dispatch_overlap: on +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6002 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 236 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114815754533Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114815754533Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2fea481b5dc123e950efab4ab037bb852652c2a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114815754533Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66873027584, + "active_bytes_current_delta": 31641572352, + "active_bytes_peak": 79681550336, + "active_bytes_peak_delta": 44215214080, + "after": { + "active_bytes_current": 66873027584, + "active_bytes_peak": 79681550336, + "allocated_bytes_current": 66873027584, + "allocated_bytes_peak": 79681550336, + "device": 0, + "inactive_split_bytes_current": 2947453952, + "inactive_split_bytes_peak": 4350796800, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 95567216640, + "reserved_bytes_peak": 95567216640, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T11:48:15.754039Z", + "world_size": 8 + }, + "allocated_bytes_current": 66873027584, + "allocated_bytes_current_delta": 31641572352, + "allocated_bytes_peak": 79681550336, + "allocated_bytes_peak_delta": 44215214080, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T11:47:06.472559Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2947453952, + "inactive_split_bytes_current_delta": 2812537856, + "inactive_split_bytes_peak": 4350796800, + "inactive_split_bytes_peak_delta": 4190164992, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 95567216640, + "reserved_bytes_current_delta": 56321114112, + "reserved_bytes_peak": 95567216640, + "reserved_bytes_peak_delta": 56321114112, + "segment_count_current": 545, + "segment_count_current_delta": 473, + "segment_count_peak": 545, + "segment_count_peak_delta": 473, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T114815754533Z.pickle", + "timestamp_utc": "2026-08-01T11:48:15.864462Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114815754533Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114815754533Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f31a4a95dce2481a90021b9db496e5f906d4c68a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T114815754533Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dfae7821ebeffc3395ece0afea396bc184cd12aa616a39a90ee3c0936ef61b3 +size 5399206 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114816692894Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114816692894Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94b77cf81b808f0303b7ea8a090f7eacf39da2cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114816692894Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66873027584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103316759552, + "active_bytes_peak_delta": 23635209216, + "after": { + "active_bytes_current": 66873027584, + "active_bytes_peak": 103316759552, + "allocated_bytes_current": 66873027584, + "allocated_bytes_peak": 103316759552, + "device": 0, + "inactive_split_bytes_current": 2725155840, + "inactive_split_bytes_peak": 4350796800, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 105138618368, + "reserved_bytes_peak": 105138618368, + "segment_count_current": 611, + "segment_count_peak": 611, + "timestamp_utc": "2026-08-01T11:48:16.692478Z", + "world_size": 8 + }, + "allocated_bytes_current": 66873027584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103316759552, + "allocated_bytes_peak_delta": 23635209216, + "before": { + "active_bytes_current": 66873027584, + "active_bytes_peak": 79681550336, + "allocated_bytes_current": 66873027584, + "allocated_bytes_peak": 79681550336, + "device": 0, + "inactive_split_bytes_current": 2947453952, + "inactive_split_bytes_peak": 4350796800, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 95567216640, + "reserved_bytes_peak": 95567216640, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T11:48:15.883525Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2725155840, + "inactive_split_bytes_current_delta": -222298112, + "inactive_split_bytes_peak": 4350796800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 105138618368, + "reserved_bytes_current_delta": 9571401728, + "reserved_bytes_peak": 105138618368, + "reserved_bytes_peak_delta": 9571401728, + "segment_count_current": 611, + "segment_count_current_delta": 66, + "segment_count_peak": 611, + "segment_count_peak_delta": 66, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T114816692894Z.pickle", + "timestamp_utc": "2026-08-01T11:48:16.800084Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114816692894Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114816692894Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8d10e5b1b9c159ad9b70e13e449e32e8f098dda9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T114816692894Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:378b6291a579fb6940941d5766a64e1006c2e903ae2f9e8c1e50aaea5301bcaa +size 5597535 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114817621287Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114817621287Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7aef91f47a5ef765d8b04f4a6f653a6250919342 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114817621287Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66873027584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103318223360, + "active_bytes_peak_delta": 1463808, + "after": { + "active_bytes_current": 66873027584, + "active_bytes_peak": 103318223360, + "allocated_bytes_current": 66873027584, + "allocated_bytes_peak": 103318223360, + "device": 0, + "inactive_split_bytes_current": 2729350144, + "inactive_split_bytes_peak": 4350796800, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 105138618368, + "reserved_bytes_peak": 105138618368, + "segment_count_current": 611, + "segment_count_peak": 611, + "timestamp_utc": "2026-08-01T11:48:17.620878Z", + "world_size": 8 + }, + "allocated_bytes_current": 66873027584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103318223360, + "allocated_bytes_peak_delta": 1463808, + "before": { + "active_bytes_current": 66873027584, + "active_bytes_peak": 103316759552, + "allocated_bytes_current": 66873027584, + "allocated_bytes_peak": 103316759552, + "device": 0, + "inactive_split_bytes_current": 2725155840, + "inactive_split_bytes_peak": 4350796800, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 105138618368, + "reserved_bytes_peak": 105138618368, + "segment_count_current": 611, + "segment_count_peak": 611, + "timestamp_utc": "2026-08-01T11:48:16.815760Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2729350144, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 4350796800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 105138618368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105138618368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 611, + "segment_count_current_delta": 0, + "segment_count_peak": 611, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T114817621287Z.pickle", + "timestamp_utc": "2026-08-01T11:48:17.725948Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114817621287Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114817621287Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f826fa67c6f166cd04df773966f1cc468ff266bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T114817621287Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4dc000da4f49961108171d41ba5d5ae0910a26825ba75358783dc7243a50171 +size 5599491 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114820284456Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114820284456Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70b0a7107aa428750ffadaa26a1ea56be510e5fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114820284456Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890988544, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 155308953600, + "active_bytes_peak_delta": 51990730240, + "after": { + "active_bytes_current": 66890988544, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66890988544, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_peak": 6026177536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_peak": 165400281088, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:48:20.283904Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890988544, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103332327424, + "allocated_bytes_peak_delta": 14104064, + "before": { + "active_bytes_current": 66873027584, + "active_bytes_peak": 103318223360, + "allocated_bytes_current": 66873027584, + "allocated_bytes_peak": 103318223360, + "device": 0, + "inactive_split_bytes_current": 2729350144, + "inactive_split_bytes_peak": 4350796800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 105138618368, + "reserved_bytes_peak": 105138618368, + "segment_count_current": 611, + "segment_count_peak": 611, + "timestamp_utc": "2026-08-01T11:48:17.741023Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_current_delta": 210628608, + "inactive_split_bytes_peak": 6026177536, + "inactive_split_bytes_peak_delta": 1675380736, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_current_delta": 60261662720, + "reserved_bytes_peak": 165400281088, + "reserved_bytes_peak_delta": 60261662720, + "segment_count_current": 967, + "segment_count_current_delta": 356, + "segment_count_peak": 967, + "segment_count_peak_delta": 356, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T114820284456Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.406287Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114820284456Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114820284456Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d5845b049c59e8c5b7855be1789f118e5a535256 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T114820284456Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f48d6342133527d6563112f662314f7f31370be0e0b95caee176f21bf48629d +size 6632701 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114820969194Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114820969194Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f28d679ea027c7a5c9b10b4b31ceaf220cdf8bb4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114820969194Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890988544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155308953600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890988544, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66890988544, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_peak": 6026177536, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_peak": 165400281088, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:48:20.968712Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890988544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103332327424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890988544, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66890988544, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_peak": 6026177536, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_peak": 165400281088, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:48:20.422967Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6026177536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165400281088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 967, + "segment_count_current_delta": 0, + "segment_count_peak": 967, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T114820969194Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.977180Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114820969194Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114820969194Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e73ee8c764420a2782034fe46d8f0050f852a06d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T114820969194Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7614e3cf2ce887b63c0abcdef774bbd4cef24ccb105a661cf776eea0e422fd0 +size 298923 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114821516596Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114821516596Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4c7ff8bba7915af1bf6aaeba2f8d7862b23e06cf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114821516596Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890988544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155308953600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890988544, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66890988544, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_peak": 6026177536, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_peak": 165400281088, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:48:21.516147Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890988544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103332327424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890988544, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66890988544, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_peak": 6026177536, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_peak": 165400281088, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:48:20.978176Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6026177536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165400281088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 967, + "segment_count_current_delta": 0, + "segment_count_peak": 967, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T114821516596Z.pickle", + "timestamp_utc": "2026-08-01T11:48:21.524129Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114821516596Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114821516596Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca58c554ee9dcb29cbd3930502ceb97f51616662 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T114821516596Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6719486f8b883b5fe2b7553a8b1ff7de5517e98cb1b56c73ac8ede3a284c6a8e +size 298917 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114822059976Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114822059976Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6860e2f123c0cdbc24aebf3235813027a9a4d3f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114822059976Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890988544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155308953600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890988544, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66890988544, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_peak": 6026177536, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_peak": 165400281088, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:48:22.059594Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890988544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103332327424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890988544, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66890988544, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_peak": 6026177536, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_peak": 165400281088, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:48:21.525113Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6026177536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165400281088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 967, + "segment_count_current_delta": 0, + "segment_count_peak": 967, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T114822059976Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.066777Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114822059976Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114822059976Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d01842cc9b072bff36423059db8cfb57a274cdf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T114822059976Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc903bc6e8151ce68b6bf8c449cebd432ba1168b78dbe5523511c5bc7e407af2 +size 298923 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114822604081Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114822604081Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1471c89cfea000ee8030b5ef99ec576527f84ec0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114822604081Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890988544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155308953600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890988544, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66890988544, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_peak": 6026177536, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_peak": 165400281088, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:48:22.603694Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890988544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103332327424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890988544, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66890988544, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_peak": 6026177536, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_peak": 165400281088, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:48:22.067727Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6026177536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165400281088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 967, + "segment_count_current_delta": 0, + "segment_count_peak": 967, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T114822604081Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.610962Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114822604081Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114822604081Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d8a9148d5de425296730c2b020fc498cfb1ab580 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T114822604081Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb54334793a6e2ceee24c9d99d7d5ad6346eda2ab5f288a6600e36f0887d757c +size 298917 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114823148171Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114823148171Z.json new file mode 100644 index 0000000000000000000000000000000000000000..693d7631f152ba67ed7f6dcd77bf54873080c99b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114823148171Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890988544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155308953600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890988544, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66890988544, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_peak": 6026177536, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_peak": 165400281088, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:48:23.147792Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890988544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103332327424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890988544, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66890988544, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_peak": 6026177536, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_peak": 165400281088, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:48:22.611955Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6026177536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165400281088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 967, + "segment_count_current_delta": 0, + "segment_count_peak": 967, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T114823148171Z.pickle", + "timestamp_utc": "2026-08-01T11:48:23.154875Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114823148171Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114823148171Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..efa27777dc690f7ca4366bf0f40d15b783789a44 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T114823148171Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5658cbc95239b8466f55c6c1f20da3c4f2e0f80141a670e313ed0334fa4853f +size 298923 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114917758429Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114917758429Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42df07b5d6c0c7beba812c40bd5c62dcfefb8c52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114917758429Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66857354240, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 155308953600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66857354240, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66857354240, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2682108928, + "inactive_split_bytes_peak": 6026177536, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 166306250752, + "reserved_bytes_peak": 166306250752, + "segment_count_current": 972, + "segment_count_peak": 972, + "timestamp_utc": "2026-08-01T11:49:17.757948Z", + "world_size": 8 + }, + "allocated_bytes_current": 66857354240, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103332327424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890988544, + "active_bytes_peak": 155308953600, + "allocated_bytes_current": 66890988544, + "allocated_bytes_peak": 103332327424, + "device": 0, + "inactive_split_bytes_current": 2939978752, + "inactive_split_bytes_peak": 6026177536, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 165400281088, + "reserved_bytes_peak": 165400281088, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:48:23.155848Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2682108928, + "inactive_split_bytes_current_delta": -257869824, + "inactive_split_bytes_peak": 6026177536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 166306250752, + "reserved_bytes_current_delta": 905969664, + "reserved_bytes_peak": 166306250752, + "reserved_bytes_peak_delta": 905969664, + "segment_count_current": 972, + "segment_count_current_delta": 5, + "segment_count_peak": 972, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T114917758429Z.pickle", + "timestamp_utc": "2026-08-01T11:49:17.842675Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114917758429Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114917758429Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..475e96dc430bc393169daf6d96223bd0aebad17a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T114917758429Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eacedf0c18d89851a3b6d303e79a3acdcda002c38c011758edab5010b50528a +size 4242618 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114815754645Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114815754645Z.json new file mode 100644 index 0000000000000000000000000000000000000000..35da2f653dc3d3110dc3e82308eea104d5724a63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114815754645Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 31640019968, + "active_bytes_peak": 79684781568, + "active_bytes_peak_delta": 44218445312, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79684781568, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79684781568, + "device": 1, + "inactive_split_bytes_current": 3156624384, + "inactive_split_bytes_peak": 3753853952, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 96267665408, + "reserved_bytes_peak": 96267665408, + "segment_count_current": 552, + "segment_count_peak": 552, + "timestamp_utc": "2026-08-01T11:48:15.754006Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 31640019968, + "allocated_bytes_peak": 79684781568, + "allocated_bytes_peak_delta": 44218445312, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:47:06.472850Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3156624384, + "inactive_split_bytes_current_delta": 3023805440, + "inactive_split_bytes_peak": 3753853952, + "inactive_split_bytes_peak_delta": 3593467904, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 96267665408, + "reserved_bytes_current_delta": 57023660032, + "reserved_bytes_peak": 96267665408, + "reserved_bytes_peak_delta": 57023660032, + "segment_count_current": 552, + "segment_count_current_delta": 481, + "segment_count_peak": 552, + "segment_count_peak_delta": 481, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T114815754645Z.pickle", + "timestamp_utc": "2026-08-01T11:48:15.866155Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114815754645Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114815754645Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f00c9a71b1d30b515bc697b23b2253c0d00151fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T114815754645Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f319ea3f17278d4e34e668dcdd7a59fa64870ebfadd569f5cb7595143aefa2d +size 5392636 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114816692835Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114816692835Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9b72fd23c04e81995e1a48c45dcc714b1186889a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114816692835Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322202112, + "active_bytes_peak_delta": 23637420544, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322202112, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322202112, + "device": 1, + "inactive_split_bytes_current": 2699445248, + "inactive_split_bytes_peak": 3753853952, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 105155395584, + "reserved_bytes_peak": 105155395584, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T11:48:16.692422Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322202112, + "allocated_bytes_peak_delta": 23637420544, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79684781568, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79684781568, + "device": 1, + "inactive_split_bytes_current": 3156624384, + "inactive_split_bytes_peak": 3753853952, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 96267665408, + "reserved_bytes_peak": 96267665408, + "segment_count_current": 552, + "segment_count_peak": 552, + "timestamp_utc": "2026-08-01T11:48:15.885788Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2699445248, + "inactive_split_bytes_current_delta": -457179136, + "inactive_split_bytes_peak": 3753853952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 105155395584, + "reserved_bytes_current_delta": 8887730176, + "reserved_bytes_peak": 105155395584, + "reserved_bytes_peak_delta": 8887730176, + "segment_count_current": 608, + "segment_count_current_delta": 56, + "segment_count_peak": 608, + "segment_count_peak_delta": 56, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T114816692835Z.pickle", + "timestamp_utc": "2026-08-01T11:48:16.799319Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114816692835Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114816692835Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ebc3ca164ed9f6eaa32f249f911450ccb6c50600 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T114816692835Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84e4d28edf45c22949ab3aa90253891609bd56ccffb0e2e9bcc9ef814ad9f393 +size 5597427 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114817621277Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114817621277Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5d8d6a4f38120d5fd7c660460ceeef8e15bd338 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114817621277Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322202112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322202112, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322202112, + "device": 1, + "inactive_split_bytes_current": 2703639552, + "inactive_split_bytes_peak": 3753853952, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 105155395584, + "reserved_bytes_peak": 105155395584, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T11:48:17.620877Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322202112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322202112, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322202112, + "device": 1, + "inactive_split_bytes_current": 2699445248, + "inactive_split_bytes_peak": 3753853952, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 105155395584, + "reserved_bytes_peak": 105155395584, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T11:48:16.814673Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2703639552, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 3753853952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 105155395584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105155395584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 608, + "segment_count_current_delta": 0, + "segment_count_peak": 608, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T114817621277Z.pickle", + "timestamp_utc": "2026-08-01T11:48:17.724369Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114817621277Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114817621277Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3591c744f05e871261eb6ed50c5cb59b0d3e6ea1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T114817621277Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f729c7e10537c604b083adff4df665aede26ead872920c610d970f63298e411 +size 5599372 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114820284471Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114820284471Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c481064d694317f4d86f8c7c2d0448941a8097e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114820284471Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 51981004800, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.283895Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 5688832, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322202112, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322202112, + "device": 1, + "inactive_split_bytes_current": 2703639552, + "inactive_split_bytes_peak": 3753853952, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 105155395584, + "reserved_bytes_peak": 105155395584, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T11:48:17.739207Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_current_delta": 208531456, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 2183706624, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 60217622528, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 60217622528, + "segment_count_current": 959, + "segment_count_current_delta": 351, + "segment_count_peak": 959, + "segment_count_peak_delta": 351, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T114820284471Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.405952Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114820284471Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114820284471Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fed4eb5ea371b5550554f6a8e133d6c7fd694329 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T114820284471Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:066df4c26efeeb277f944b31e0dd46bf8b2fcb4814eb14bec4ed250081364d81 +size 6614581 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114820969156Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114820969156Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0e9eabe521fbc0574164846d94da13237df72665 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114820969156Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.968701Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.422635Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T114820969156Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.976918Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114820969156Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114820969156Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..818aa712fe4eb9cc9e9a8f0ef11a0ee674998a44 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T114820969156Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:201f21ea22bae21d7da173fab1a447feeedc5510e777b2bd81e0c5118882e21b +size 293433 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114821516528Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114821516528Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eda2723a180b736a0f803ef3dff4719a2494cc0f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114821516528Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:21.516122Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.977910Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T114821516528Z.pickle", + "timestamp_utc": "2026-08-01T11:48:21.523441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114821516528Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114821516528Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e08747e05ce50b3935517cb116550b86878d0b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T114821516528Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f877f1e70cbdd516fccf3f7a59b4a6eee2dec4cb7671e85741f505b4f21a1afc +size 293433 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114822059989Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114822059989Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5812a8ecd5c681752bb78e6c2e473c39e66cedc6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114822059989Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.059608Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:21.524416Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T114822059989Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.066616Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114822059989Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114822059989Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..99e136e2e3fc4b73955a446f6940a81dd9fd98a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T114822059989Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:050ae4fe095bc8b91272c836f9ef7c362b7c521678c07d4ea7060689907112d0 +size 293433 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114822604085Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114822604085Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db141c6fcd37269324f198f0afd1ef28598531b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114822604085Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.603688Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.067570Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T114822604085Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.610779Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114822604085Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114822604085Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8241001b5e48a87b66ff1c1fdf164cbc91986ccd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T114822604085Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba5c0714db243f940d2ba680ec232d8462973fd82aea600249d13bc4bab3f0b8 +size 293433 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114823148186Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114823148186Z.json new file mode 100644 index 0000000000000000000000000000000000000000..838d597581997c6881a1dce6664ef5e1460a7e15 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114823148186Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:23.147803Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.611734Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T114823148186Z.pickle", + "timestamp_utc": "2026-08-01T11:48:23.154900Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114823148186Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114823148186Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6604fe2c21be1b4376e36f1984b09fee070b87a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T114823148186Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb2776c111a33256c10e7345dbed252dc7c45e917bcc6264eec504c1d3151c8b +size 293433 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114917758462Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114917758462Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f25fc51551d682e086b77a44ed9fe7438ba6da9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114917758462Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855801856, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855801856, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66855801856, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2656398336, + "inactive_split_bytes_peak": 5937560576, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 166278987776, + "reserved_bytes_peak": 166278987776, + "segment_count_current": 964, + "segment_count_peak": 964, + "timestamp_utc": "2026-08-01T11:49:17.757947Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855801856, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 1, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:23.155853Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2656398336, + "inactive_split_bytes_current_delta": -255772672, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 166278987776, + "reserved_bytes_current_delta": 905969664, + "reserved_bytes_peak": 166278987776, + "reserved_bytes_peak_delta": 905969664, + "segment_count_current": 964, + "segment_count_current_delta": 5, + "segment_count_peak": 964, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T114917758462Z.pickle", + "timestamp_utc": "2026-08-01T11:49:17.850561Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114917758462Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114917758462Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d403ea3453e5def75b9fc7016718dafdbccdae9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T114917758462Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c7293a72d32516cc8a61d5e0f1724deccc0dfd77b1b32ef3565c60373cfe2d2 +size 4100331 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114815754758Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114815754758Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5333ead37fb9429bf7b2d71984474619cee3e465 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114815754758Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 31640019968, + "active_bytes_peak": 79684781568, + "active_bytes_peak_delta": 44218445312, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79684781568, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79684781568, + "device": 2, + "inactive_split_bytes_current": 3156624384, + "inactive_split_bytes_peak": 3871376384, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 96737427456, + "reserved_bytes_peak": 96737427456, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T11:48:15.754115Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 31640019968, + "allocated_bytes_peak": 79684781568, + "allocated_bytes_peak_delta": 44218445312, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:47:06.472929Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3156624384, + "inactive_split_bytes_current_delta": 3023805440, + "inactive_split_bytes_peak": 3871376384, + "inactive_split_bytes_peak_delta": 3710990336, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 96737427456, + "reserved_bytes_current_delta": 57493422080, + "reserved_bytes_peak": 96737427456, + "reserved_bytes_peak_delta": 57493422080, + "segment_count_current": 554, + "segment_count_current_delta": 483, + "segment_count_peak": 554, + "segment_count_peak_delta": 483, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T114815754758Z.pickle", + "timestamp_utc": "2026-08-01T11:48:15.870994Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114815754758Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114815754758Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7489d608f81fab42fddf5593169fa28e475a7812 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T114815754758Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a9b1544b7b4926e8ec2a5c61cb7070a6bb2b38351aad24ea0580d778a761adf +size 5392970 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114816692795Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114816692795Z.json new file mode 100644 index 0000000000000000000000000000000000000000..68e2bf5466d6ca392b670b3babd0f9b335fcdbdf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114816692795Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322202112, + "active_bytes_peak_delta": 23637420544, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322202112, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322202112, + "device": 2, + "inactive_split_bytes_current": 2699445248, + "inactive_split_bytes_peak": 3871376384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106329800704, + "reserved_bytes_peak": 106329800704, + "segment_count_current": 613, + "segment_count_peak": 613, + "timestamp_utc": "2026-08-01T11:48:16.692392Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322202112, + "allocated_bytes_peak_delta": 23637420544, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79684781568, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79684781568, + "device": 2, + "inactive_split_bytes_current": 3156624384, + "inactive_split_bytes_peak": 3871376384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 96737427456, + "reserved_bytes_peak": 96737427456, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T11:48:15.890016Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2699445248, + "inactive_split_bytes_current_delta": -457179136, + "inactive_split_bytes_peak": 3871376384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106329800704, + "reserved_bytes_current_delta": 9592373248, + "reserved_bytes_peak": 106329800704, + "reserved_bytes_peak_delta": 9592373248, + "segment_count_current": 613, + "segment_count_current_delta": 59, + "segment_count_peak": 613, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T114816692795Z.pickle", + "timestamp_utc": "2026-08-01T11:48:16.800018Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114816692795Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114816692795Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ac735b08cde70df705441642aa1fda874712d03 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T114816692795Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eac69b4d0e2bcf8af8a14d2b08e904c5457e2681830130ba3e9e4f0865442073 +size 5598220 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114817621278Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114817621278Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0456c60e046aa3a3d1511ffca330f98d58f0aa73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114817621278Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322202112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322202112, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322202112, + "device": 2, + "inactive_split_bytes_current": 2703639552, + "inactive_split_bytes_peak": 3871376384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106329800704, + "reserved_bytes_peak": 106329800704, + "segment_count_current": 613, + "segment_count_peak": 613, + "timestamp_utc": "2026-08-01T11:48:17.620878Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322202112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322202112, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322202112, + "device": 2, + "inactive_split_bytes_current": 2699445248, + "inactive_split_bytes_peak": 3871376384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106329800704, + "reserved_bytes_peak": 106329800704, + "segment_count_current": 613, + "segment_count_peak": 613, + "timestamp_utc": "2026-08-01T11:48:16.815182Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2703639552, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 3871376384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106329800704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106329800704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 613, + "segment_count_current_delta": 0, + "segment_count_peak": 613, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T114817621278Z.pickle", + "timestamp_utc": "2026-08-01T11:48:17.724744Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114817621278Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114817621278Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2de9b4de03f35ff999e73aaced4d755d4c7c81b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T114817621278Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7acd06059af2d9999debd9aad117797adc2c226bb9ae0d65e9edd060d40fbefa +size 5599591 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114820284492Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114820284492Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e8c9c213dce9f0fb07661636275c17fdf1b30aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114820284492Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 51981004800, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.283921Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 5688832, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322202112, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322202112, + "device": 2, + "inactive_split_bytes_current": 2703639552, + "inactive_split_bytes_peak": 3871376384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106329800704, + "reserved_bytes_peak": 106329800704, + "segment_count_current": 613, + "segment_count_peak": 613, + "timestamp_utc": "2026-08-01T11:48:17.739341Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_current_delta": 208531456, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 2066184192, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 59043217408, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 59043217408, + "segment_count_current": 959, + "segment_count_current_delta": 346, + "segment_count_peak": 959, + "segment_count_peak_delta": 346, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T114820284492Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.406488Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114820284492Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114820284492Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a85740e64f6a12de7e222db006ee640c8f42d2b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T114820284492Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f188b3568b92ba1627c27b2b4b8aba96f9af98cd3749ca79510556a97f92ab39 +size 6614477 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T114820969192Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T114820969192Z.json new file mode 100644 index 0000000000000000000000000000000000000000..00032b052903b5d3eedfe21b75bd579ba007f80d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T114820969192Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.968696Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.422675Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T114820969192Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.976921Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T114820969192Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T114820969192Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..591d74e52056d1a98e50bc484f867c52d5e8be79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T114820969192Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:917c83083aa70bb0838e6874380576e6d1351023c37d3afe2001343b7625cf5e +size 293370 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T114821516503Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T114821516503Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a31359b0c4b68fea3cc88ff38b5ceeaf3616485 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T114821516503Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:21.516093Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.977952Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T114821516503Z.pickle", + "timestamp_utc": "2026-08-01T11:48:21.523430Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T114821516503Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T114821516503Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fc46faba5f7b135c00ecb3a46a7c77a1c3799d57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T114821516503Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eff265420a6d418adc9958650f77f6f24e72588bca26c1aaaca7ffbd6723adf5 +size 293339 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T114822059955Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T114822059955Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4bf7a405474e80c75226fe47ca91fb91066deb40 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T114822059955Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.059568Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:21.524394Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T114822059955Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.066406Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T114822059955Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T114822059955Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c6f7cd853132a2e334a4f8a980ed7f5527353701 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T114822059955Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4576bf9e675cb3ddadb9f41f32dbbe7557f737265b86d664912e584dfb0490e +size 293370 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T114822604101Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T114822604101Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3c0304872e98e71a56483b0b14dd5ca22872b22b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T114822604101Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.603691Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.067345Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T114822604101Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.610912Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T114822604101Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T114822604101Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f1f2868c4b87b0e6cc4093a400ada44deda61eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T114822604101Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eebc47523c2984d0d7846bea0af70888c48459b658911a3703f6d4698e1d9b3c +size 293339 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T114823148140Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T114823148140Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7bdb2922eed0f890c327f0bcd77554eb2b8f7251 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T114823148140Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:23.147756Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.611887Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T114823148140Z.pickle", + "timestamp_utc": "2026-08-01T11:48:23.154657Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T114823148140Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T114823148140Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76ab3bab146eb03fb38b2c6863be6e94de17cecf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T114823148140Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7adfc29ed612ffdc69d7791172895897c63958d6b64f6a5a79b19f280db95bb +size 293370 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T114917758462Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T114917758462Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65386294319e105660935285fb52704414bec7d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T114917758462Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855801856, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855801856, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66855801856, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2656398336, + "inactive_split_bytes_peak": 5937560576, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 167453392896, + "reserved_bytes_peak": 167453392896, + "segment_count_current": 969, + "segment_count_peak": 969, + "timestamp_utc": "2026-08-01T11:49:17.757966Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855801856, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 2, + "inactive_split_bytes_current": 2912171008, + "inactive_split_bytes_peak": 5937560576, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:23.155590Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2656398336, + "inactive_split_bytes_current_delta": -255772672, + "inactive_split_bytes_peak": 5937560576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 167453392896, + "reserved_bytes_current_delta": 2080374784, + "reserved_bytes_peak": 167453392896, + "reserved_bytes_peak_delta": 2080374784, + "segment_count_current": 969, + "segment_count_current_delta": 10, + "segment_count_peak": 969, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T114917758462Z.pickle", + "timestamp_utc": "2026-08-01T11:49:17.853332Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T114917758462Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T114917758462Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a8a46f4d4bb4f6cf67617df0a46e1d1085b2ae3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T114917758462Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bca3d26c9faa6f4d0187c45070b55303a91db166816436933d9b7940c9999c3 +size 4094670 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T114815754600Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T114815754600Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd0cfb5fb071a7294f90143446392c9036f3a5bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T114815754600Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 31640019968, + "active_bytes_peak": 79684781568, + "active_bytes_peak_delta": 44218445312, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79684781568, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79684781568, + "device": 3, + "inactive_split_bytes_current": 3156624384, + "inactive_split_bytes_peak": 3756708864, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 95797903360, + "reserved_bytes_peak": 95797903360, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T11:48:15.754009Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 31640019968, + "allocated_bytes_peak": 79684781568, + "allocated_bytes_peak_delta": 44218445312, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:47:06.472986Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3156624384, + "inactive_split_bytes_current_delta": 3023805440, + "inactive_split_bytes_peak": 3756708864, + "inactive_split_bytes_peak_delta": 3596322816, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 95797903360, + "reserved_bytes_current_delta": 56553897984, + "reserved_bytes_peak": 95797903360, + "reserved_bytes_peak_delta": 56553897984, + "segment_count_current": 545, + "segment_count_current_delta": 474, + "segment_count_peak": 545, + "segment_count_peak_delta": 474, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T114815754600Z.pickle", + "timestamp_utc": "2026-08-01T11:48:15.865469Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T114815754600Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T114815754600Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4bce043d4425b4424751105e668410cda99f32b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T114815754600Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c17206978be146f3f2a304565d89e898b708582126df9766312836ab2d7b8da +size 5391501 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T114816692810Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T114816692810Z.json new file mode 100644 index 0000000000000000000000000000000000000000..07581f0613b122c6e4051b0d0e27240e1ec79cab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T114816692810Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103325860864, + "active_bytes_peak_delta": 23641079296, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103325860864, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103325860864, + "device": 3, + "inactive_split_bytes_current": 2934326272, + "inactive_split_bytes_peak": 3756708864, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 105390276608, + "reserved_bytes_peak": 105390276608, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T11:48:16.692402Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103325860864, + "allocated_bytes_peak_delta": 23641079296, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79684781568, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79684781568, + "device": 3, + "inactive_split_bytes_current": 3156624384, + "inactive_split_bytes_peak": 3756708864, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 95797903360, + "reserved_bytes_peak": 95797903360, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T11:48:15.881867Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2934326272, + "inactive_split_bytes_current_delta": -222298112, + "inactive_split_bytes_peak": 3756708864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 105390276608, + "reserved_bytes_current_delta": 9592373248, + "reserved_bytes_peak": 105390276608, + "reserved_bytes_peak_delta": 9592373248, + "segment_count_current": 608, + "segment_count_current_delta": 63, + "segment_count_peak": 608, + "segment_count_peak_delta": 63, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T114816692810Z.pickle", + "timestamp_utc": "2026-08-01T11:48:16.796307Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T114816692810Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T114816692810Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7c7b77b73b1de73c68077cd44cf2da350579e909 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T114816692810Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:555bc13dd68c6b4d1fa74d2210de27863b774d35381d74ae8a2bc3349fade7b3 +size 5597403 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T114817621273Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T114817621273Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a4763f2b671f20511125fa483175e35f6e10537 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T114817621273Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103325860864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103325860864, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103325860864, + "device": 3, + "inactive_split_bytes_current": 2703639552, + "inactive_split_bytes_peak": 3756708864, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 105390276608, + "reserved_bytes_peak": 105390276608, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T11:48:17.620874Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103325860864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103325860864, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103325860864, + "device": 3, + "inactive_split_bytes_current": 2934326272, + "inactive_split_bytes_peak": 3756708864, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 105390276608, + "reserved_bytes_peak": 105390276608, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T11:48:16.808863Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2703639552, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 3756708864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 105390276608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105390276608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 608, + "segment_count_current_delta": 0, + "segment_count_peak": 608, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T114817621273Z.pickle", + "timestamp_utc": "2026-08-01T11:48:17.716504Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T114817621273Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T114817621273Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2926270e2b9ab358eb71d61a7759936e84d7e3a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T114817621273Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6946ecf943d5d5e9a080f784e1e41510e2c4763894501b1849c39f55aa16afe2 +size 5599075 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T114820284433Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T114820284433Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2eb1cc02d12b2ac2147c5cc066f5cb1a11bba3fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T114820284433Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 155303731200, + "active_bytes_peak_delta": 51977870336, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 5847534080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 165431738368, + "reserved_bytes_peak": 165431738368, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.283895Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103329463808, + "allocated_bytes_peak_delta": 3602944, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103325860864, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103325860864, + "device": 3, + "inactive_split_bytes_current": 2703639552, + "inactive_split_bytes_peak": 3756708864, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 105390276608, + "reserved_bytes_peak": 105390276608, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T11:48:17.727797Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_current_delta": 210628608, + "inactive_split_bytes_peak": 5847534080, + "inactive_split_bytes_peak_delta": 2090825216, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 165431738368, + "reserved_bytes_current_delta": 60041461760, + "reserved_bytes_peak": 165431738368, + "reserved_bytes_peak_delta": 60041461760, + "segment_count_current": 959, + "segment_count_current_delta": 351, + "segment_count_peak": 959, + "segment_count_peak_delta": 351, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T114820284433Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.402906Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T114820284433Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T114820284433Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3e11452beb2832a4c2981dc4cae9013d0d45117 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T114820284433Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9820a6d70659f3fc24af5fa2cfcba18ddb4aed14f76637ca951b6bf4ed51f63 +size 6614394 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T114820969142Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T114820969142Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2faef69806432ab95c6244331ec4cd441bc08036 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T114820969142Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303731200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 5847534080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 165431738368, + "reserved_bytes_peak": 165431738368, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.968699Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329463808, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 5847534080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 165431738368, + "reserved_bytes_peak": 165431738368, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.418659Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5847534080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 165431738368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165431738368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T114820969142Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.975988Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T114820969142Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T114820969142Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..297759f81db305a4d69f0ab41e361d1113565205 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T114820969142Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d7d18b73b6b0f9916fd97750367d016c404d01c51cda9c9ad00e25424735351 +size 294325 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T114821516499Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T114821516499Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9996ebfcb943a117696b147e832e387462d8f4e1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T114821516499Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303731200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 5847534080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 165431738368, + "reserved_bytes_peak": 165431738368, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:21.516108Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329463808, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 5847534080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 165431738368, + "reserved_bytes_peak": 165431738368, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.976989Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5847534080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 165431738368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165431738368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T114821516499Z.pickle", + "timestamp_utc": "2026-08-01T11:48:21.523136Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T114821516499Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T114821516499Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b9d94421cb510d6cdc29adfbb62500e949f4f30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T114821516499Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f79fe5ac9981bbabf8db5813e7e81f70a8eab120d141d70e0d5d37ef352dc086 +size 294325 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T114822059985Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T114822059985Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70975d15e6f19b118b55b296a79d514f79f28a8d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T114822059985Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303731200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 5847534080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 165431738368, + "reserved_bytes_peak": 165431738368, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.059595Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329463808, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 5847534080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 165431738368, + "reserved_bytes_peak": 165431738368, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:21.524098Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5847534080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 165431738368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165431738368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T114822059985Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.066286Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T114822059985Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T114822059985Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..47edd60f9bb3ad49fb49c98f381224afc57c13d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T114822059985Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:409e1036940281fa227aec16e6481fbc22177353d61081906724e80c019dc692 +size 294325 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T114822604067Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T114822604067Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7acf6443c2307ba7d6edc38ea81aaae78876a46c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T114822604067Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303731200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 5847534080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 165431738368, + "reserved_bytes_peak": 165431738368, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.603682Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329463808, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 5847534080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 165431738368, + "reserved_bytes_peak": 165431738368, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.067248Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5847534080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 165431738368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165431738368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T114822604067Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.610424Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T114822604067Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T114822604067Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c62e4085ba1ba2a4a3c8d4f260d593f64b1fbd9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T114822604067Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db90ab4f46ca0748e6e851164c1dba43600d8e30b07a1cf592cd8ae9f3afee45 +size 294325 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T114823148275Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T114823148275Z.json new file mode 100644 index 0000000000000000000000000000000000000000..53092e974b79db3d5f7d35b112a8fa2f12efd9d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T114823148275Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303731200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 5847534080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 165431738368, + "reserved_bytes_peak": 165431738368, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:23.147831Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329463808, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 5847534080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 165431738368, + "reserved_bytes_peak": 165431738368, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.611376Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5847534080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 165431738368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165431738368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T114823148275Z.pickle", + "timestamp_utc": "2026-08-01T11:48:23.155128Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T114823148275Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T114823148275Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..848a0865faaf4df9b37f82aee3dc90b48da7e4c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T114823148275Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7b923eba4f38a066385bad74d1ccb4c24270f86a8f672f92b62f38b0b3e0b90 +size 294325 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T114917758446Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T114917758446Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80888c527be962542d03df6cd9c5ff95b8e18abf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T114917758446Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855801856, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 155303731200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855801856, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66855801856, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2656398336, + "inactive_split_bytes_peak": 5847534080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 166337708032, + "reserved_bytes_peak": 166337708032, + "segment_count_current": 964, + "segment_count_peak": 964, + "timestamp_utc": "2026-08-01T11:49:17.757954Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855801856, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103329463808, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303731200, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329463808, + "device": 3, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 5847534080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 165431738368, + "reserved_bytes_peak": 165431738368, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:23.156086Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2656398336, + "inactive_split_bytes_current_delta": -257869824, + "inactive_split_bytes_peak": 5847534080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 166337708032, + "reserved_bytes_current_delta": 905969664, + "reserved_bytes_peak": 166337708032, + "reserved_bytes_peak_delta": 905969664, + "segment_count_current": 964, + "segment_count_current_delta": 5, + "segment_count_peak": 964, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T114917758446Z.pickle", + "timestamp_utc": "2026-08-01T11:49:17.848271Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T114917758446Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T114917758446Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..defef2038546eb27a24361bbc673fa5a54845998 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T114917758446Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92b0ec19ead4d3225f21443fd219247ea8c65f8f8460620e9a815b99d0e2bf39 +size 4101104 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T114815754583Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T114815754583Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2dc5f6c4ac6959c562a9defab710dea6bdd27379 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T114815754583Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872952832, + "active_bytes_current_delta": 31641055232, + "active_bytes_peak": 79677179904, + "active_bytes_peak_delta": 44210401280, + "after": { + "active_bytes_current": 66872952832, + "active_bytes_peak": 79677179904, + "allocated_bytes_current": 66872952832, + "allocated_bytes_peak": 79677179904, + "device": 4, + "inactive_split_bytes_current": 3169826816, + "inactive_split_bytes_peak": 3852145664, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 95800000512, + "reserved_bytes_peak": 95800000512, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T11:48:15.754122Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872952832, + "allocated_bytes_current_delta": 31641055232, + "allocated_bytes_peak": 79677179904, + "allocated_bytes_peak_delta": 44210401280, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:47:06.472720Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3169826816, + "inactive_split_bytes_current_delta": 3037450240, + "inactive_split_bytes_peak": 3852145664, + "inactive_split_bytes_peak_delta": 3698736128, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 95800000512, + "reserved_bytes_current_delta": 56555995136, + "reserved_bytes_peak": 95800000512, + "reserved_bytes_peak_delta": 56555995136, + "segment_count_current": 546, + "segment_count_current_delta": 475, + "segment_count_peak": 546, + "segment_count_peak_delta": 475, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T114815754583Z.pickle", + "timestamp_utc": "2026-08-01T11:48:15.868573Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T114815754583Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T114815754583Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ddb22bd7fa41a6f6108107b48a04e7d0919fa291 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T114815754583Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac38d24738db914b3bf62698d0f3a975cf80e3541f7ead3f57a9bf448ac32306 +size 5400220 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T114816692803Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T114816692803Z.json new file mode 100644 index 0000000000000000000000000000000000000000..85486779505c071e5d2f27555346111fa9ee6995 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T114816692803Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872952832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103321063936, + "active_bytes_peak_delta": 23643884032, + "after": { + "active_bytes_current": 66872952832, + "active_bytes_peak": 103321063936, + "allocated_bytes_current": 66872952832, + "allocated_bytes_peak": 103321063936, + "device": 4, + "inactive_split_bytes_current": 2714744832, + "inactive_split_bytes_peak": 4576842752, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 105367207936, + "reserved_bytes_peak": 105367207936, + "segment_count_current": 611, + "segment_count_peak": 611, + "timestamp_utc": "2026-08-01T11:48:16.692393Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872952832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103321063936, + "allocated_bytes_peak_delta": 23643884032, + "before": { + "active_bytes_current": 66872952832, + "active_bytes_peak": 79677179904, + "allocated_bytes_current": 66872952832, + "allocated_bytes_peak": 79677179904, + "device": 4, + "inactive_split_bytes_current": 3169826816, + "inactive_split_bytes_peak": 3852145664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 95800000512, + "reserved_bytes_peak": 95800000512, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T11:48:15.888568Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2714744832, + "inactive_split_bytes_current_delta": -455081984, + "inactive_split_bytes_peak": 4576842752, + "inactive_split_bytes_peak_delta": 724697088, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 105367207936, + "reserved_bytes_current_delta": 9567207424, + "reserved_bytes_peak": 105367207936, + "reserved_bytes_peak_delta": 9567207424, + "segment_count_current": 611, + "segment_count_current_delta": 65, + "segment_count_peak": 611, + "segment_count_peak_delta": 65, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T114816692803Z.pickle", + "timestamp_utc": "2026-08-01T11:48:16.801335Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T114816692803Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T114816692803Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..50266f0b1353c96420d978b5600c773a2fd8c096 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T114816692803Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e21b91d2d5d99bf6bfba98ca980a88972646e34f96770025ec08e6d6bad07bc +size 5656798 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T114817621271Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T114817621271Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c8cfed646de331f0626b40bd86b009aab99441f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T114817621271Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872952832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103321063936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66872952832, + "active_bytes_peak": 103321063936, + "allocated_bytes_current": 66872952832, + "allocated_bytes_peak": 103321063936, + "device": 4, + "inactive_split_bytes_current": 2716841984, + "inactive_split_bytes_peak": 4576842752, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 105367207936, + "reserved_bytes_peak": 105367207936, + "segment_count_current": 611, + "segment_count_peak": 611, + "timestamp_utc": "2026-08-01T11:48:17.620854Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872952832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103321063936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66872952832, + "active_bytes_peak": 103321063936, + "allocated_bytes_current": 66872952832, + "allocated_bytes_peak": 103321063936, + "device": 4, + "inactive_split_bytes_current": 2714744832, + "inactive_split_bytes_peak": 4576842752, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 105367207936, + "reserved_bytes_peak": 105367207936, + "segment_count_current": 611, + "segment_count_peak": 611, + "timestamp_utc": "2026-08-01T11:48:16.816780Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2716841984, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 4576842752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 105367207936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105367207936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 611, + "segment_count_current_delta": 0, + "segment_count_peak": 611, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T114817621271Z.pickle", + "timestamp_utc": "2026-08-01T11:48:17.726734Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T114817621271Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T114817621271Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d95205d50d3e5cf0708932ddee560e544fd7b08d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T114817621271Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46434f3576af234eb5c98d2337b9a891bbc425feaee1f4e34aa746fceeb2680d +size 5605633 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T114820284490Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T114820284490Z.json new file mode 100644 index 0000000000000000000000000000000000000000..808978d02a40f6685d8bf41819eaaf1e04013dd2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T114820284490Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890913792, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 155305208832, + "active_bytes_peak_delta": 51984144896, + "after": { + "active_bytes_current": 66890913792, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66890913792, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_peak": 5790593024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 165446418432, + "reserved_bytes_peak": 165446418432, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:48:20.283909Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890913792, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103330941440, + "allocated_bytes_peak_delta": 9877504, + "before": { + "active_bytes_current": 66872952832, + "active_bytes_peak": 103321063936, + "allocated_bytes_current": 66872952832, + "allocated_bytes_peak": 103321063936, + "device": 4, + "inactive_split_bytes_current": 2716841984, + "inactive_split_bytes_peak": 4576842752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 105367207936, + "reserved_bytes_peak": 105367207936, + "segment_count_current": 611, + "segment_count_peak": 611, + "timestamp_utc": "2026-08-01T11:48:17.741757Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_current_delta": 210628608, + "inactive_split_bytes_peak": 5790593024, + "inactive_split_bytes_peak_delta": 1213750272, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 165446418432, + "reserved_bytes_current_delta": 60079210496, + "reserved_bytes_peak": 165446418432, + "reserved_bytes_peak_delta": 60079210496, + "segment_count_current": 963, + "segment_count_current_delta": 352, + "segment_count_peak": 963, + "segment_count_peak_delta": 352, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T114820284490Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.407992Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T114820284490Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T114820284490Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7a2d5e12a607e43857a301794cf943f77140d41b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T114820284490Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8495874dc23f47cb140ccd2004d611fa8eceb68672604bcf982779dc245d141a +size 6630098 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T114820969142Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T114820969142Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d67d37723febe4f09eec36de3196c3355e0bd637 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T114820969142Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890913792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155305208832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890913792, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66890913792, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_peak": 5790593024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 165446418432, + "reserved_bytes_peak": 165446418432, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:48:20.968672Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890913792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103330941440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890913792, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66890913792, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_peak": 5790593024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 165446418432, + "reserved_bytes_peak": 165446418432, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:48:20.424618Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5790593024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 165446418432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165446418432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 963, + "segment_count_current_delta": 0, + "segment_count_peak": 963, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T114820969142Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.977032Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T114820969142Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T114820969142Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d07b1c48f34b441348d2d39dbba1c711151856e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T114820969142Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac343b5c2f72e83613c5444e13283c7641b79f23dd7aa82215392fa1edefe79e +size 295914 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T114821516514Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T114821516514Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e87e3a2b9f9a350cf38023ff89823cd19d59e97 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T114821516514Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890913792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155305208832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890913792, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66890913792, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_peak": 5790593024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 165446418432, + "reserved_bytes_peak": 165446418432, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:48:21.516106Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890913792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103330941440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890913792, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66890913792, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_peak": 5790593024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 165446418432, + "reserved_bytes_peak": 165446418432, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:48:20.978019Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5790593024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 165446418432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165446418432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 963, + "segment_count_current_delta": 0, + "segment_count_peak": 963, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T114821516514Z.pickle", + "timestamp_utc": "2026-08-01T11:48:21.523530Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T114821516514Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T114821516514Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..77483a59b2fdd82217803befb8115296bc9026be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T114821516514Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:add63d42152e526fdb7054e25717e8e863b7cf01ec01aea55282d5576f428e6c +size 295914 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T114822059968Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T114822059968Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a060cf21e7c33acab4b05f5ad1b6963046d882f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T114822059968Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890913792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155305208832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890913792, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66890913792, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_peak": 5790593024, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 165446418432, + "reserved_bytes_peak": 165446418432, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:48:22.059571Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890913792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103330941440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890913792, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66890913792, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_peak": 5790593024, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 165446418432, + "reserved_bytes_peak": 165446418432, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:48:21.524490Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5790593024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 165446418432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165446418432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 963, + "segment_count_current_delta": 0, + "segment_count_peak": 963, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T114822059968Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.066704Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T114822059968Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T114822059968Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d400c1d597d3e3a38cb8eae823255ce932a18da2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T114822059968Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f430d843e45a84c7a142a3e6d0e776a9fe0b9e3f5a870af40e3aab3b86afabcf +size 295914 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T114822604082Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T114822604082Z.json new file mode 100644 index 0000000000000000000000000000000000000000..584d5c839b3af00b004e0984e0b38505deb503bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T114822604082Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890913792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155305208832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890913792, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66890913792, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_peak": 5790593024, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 165446418432, + "reserved_bytes_peak": 165446418432, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:48:22.603684Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890913792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103330941440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890913792, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66890913792, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_peak": 5790593024, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 165446418432, + "reserved_bytes_peak": 165446418432, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:48:22.067663Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5790593024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 165446418432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165446418432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 963, + "segment_count_current_delta": 0, + "segment_count_peak": 963, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T114822604082Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.610917Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T114822604082Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T114822604082Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3414bd57b8c0dc86b17b2c385f5f7b66c3d8eb44 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T114822604082Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80d15cea2a6ca7d97ad043998fa6fdf43f4f2cb2d379161fee69e3efa2c3b81f +size 295914 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T114823148152Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T114823148152Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae938667d8260478ca6b682cc8acab3773718976 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T114823148152Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66890913792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155305208832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66890913792, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66890913792, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_peak": 5790593024, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 165446418432, + "reserved_bytes_peak": 165446418432, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:48:23.147760Z", + "world_size": 8 + }, + "allocated_bytes_current": 66890913792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103330941440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890913792, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66890913792, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_peak": 5790593024, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 165446418432, + "reserved_bytes_peak": 165446418432, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:48:22.611913Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5790593024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 165446418432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165446418432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 963, + "segment_count_current_delta": 0, + "segment_count_peak": 963, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T114823148152Z.pickle", + "timestamp_utc": "2026-08-01T11:48:23.154818Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T114823148152Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T114823148152Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eaa89ce108cb333837c85429dd29c723860e08e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T114823148152Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03bb07553fa5a27c20fba252b737ba31157e917504e4f6ef90ad7ff0e5b8db15 +size 295914 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T114917758429Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T114917758429Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7d886b9f5fccb2306cb32875b262af56cbc33e15 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T114917758429Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66857279488, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 155305208832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66857279488, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66857279488, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2669600768, + "inactive_split_bytes_peak": 5790593024, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 166352388096, + "reserved_bytes_peak": 166352388096, + "segment_count_current": 968, + "segment_count_peak": 968, + "timestamp_utc": "2026-08-01T11:49:17.757942Z", + "world_size": 8 + }, + "allocated_bytes_current": 66857279488, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103330941440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66890913792, + "active_bytes_peak": 155305208832, + "allocated_bytes_current": 66890913792, + "allocated_bytes_peak": 103330941440, + "device": 4, + "inactive_split_bytes_current": 2927470592, + "inactive_split_bytes_peak": 5790593024, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 165446418432, + "reserved_bytes_peak": 165446418432, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:48:23.155785Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2669600768, + "inactive_split_bytes_current_delta": -257869824, + "inactive_split_bytes_peak": 5790593024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 166352388096, + "reserved_bytes_current_delta": 905969664, + "reserved_bytes_peak": 166352388096, + "reserved_bytes_peak_delta": 905969664, + "segment_count_current": 968, + "segment_count_current_delta": 5, + "segment_count_peak": 968, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T114917758429Z.pickle", + "timestamp_utc": "2026-08-01T11:49:17.842932Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T114917758429Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T114917758429Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..879a3aa85eb16483599722d398c22e29ba2001ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T114917758429Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e22459cbad5832d75e0a42b8ea7e4b04d76865e69a54b1e5e8e0679cbf4eb57 +size 4240829 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T114815754903Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T114815754903Z.json new file mode 100644 index 0000000000000000000000000000000000000000..78288a81f28a9a6bfcea490bff41f4b202890f14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T114815754903Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 31640019968, + "active_bytes_peak": 79684781568, + "active_bytes_peak_delta": 44218445312, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79684781568, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79684781568, + "device": 5, + "inactive_split_bytes_current": 2921743360, + "inactive_split_bytes_peak": 3871376384, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 95797903360, + "reserved_bytes_peak": 95797903360, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T11:48:15.754292Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 31640019968, + "allocated_bytes_peak": 79684781568, + "allocated_bytes_peak_delta": 44218445312, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:47:06.473042Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2921743360, + "inactive_split_bytes_current_delta": 2788924416, + "inactive_split_bytes_peak": 3871376384, + "inactive_split_bytes_peak_delta": 3710990336, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 95797903360, + "reserved_bytes_current_delta": 56553897984, + "reserved_bytes_peak": 95797903360, + "reserved_bytes_peak_delta": 56553897984, + "segment_count_current": 550, + "segment_count_current_delta": 479, + "segment_count_peak": 550, + "segment_count_peak_delta": 479, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T114815754903Z.pickle", + "timestamp_utc": "2026-08-01T11:48:15.866027Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T114815754903Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T114815754903Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..658c38153d731f0004e00da91eb49e8fae74900a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T114815754903Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:571e2f9cabaab873a54492a460a4fcccb9dd49fb490743dca490ebbc85947f1a +size 5392252 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T114816692912Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T114816692912Z.json new file mode 100644 index 0000000000000000000000000000000000000000..100a83b398ba9570e65f07958f0e3917946c9d8f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T114816692912Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322202112, + "active_bytes_peak_delta": 23637420544, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322202112, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322202112, + "device": 5, + "inactive_split_bytes_current": 2699445248, + "inactive_split_bytes_peak": 3871376384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104685633536, + "reserved_bytes_peak": 104685633536, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T11:48:16.692505Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322202112, + "allocated_bytes_peak_delta": 23637420544, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79684781568, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79684781568, + "device": 5, + "inactive_split_bytes_current": 2921743360, + "inactive_split_bytes_peak": 3871376384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 95797903360, + "reserved_bytes_peak": 95797903360, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T11:48:15.882351Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2699445248, + "inactive_split_bytes_current_delta": -222298112, + "inactive_split_bytes_peak": 3871376384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104685633536, + "reserved_bytes_current_delta": 8887730176, + "reserved_bytes_peak": 104685633536, + "reserved_bytes_peak_delta": 8887730176, + "segment_count_current": 606, + "segment_count_current_delta": 56, + "segment_count_peak": 606, + "segment_count_peak_delta": 56, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T114816692912Z.pickle", + "timestamp_utc": "2026-08-01T11:48:16.794123Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T114816692912Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T114816692912Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5001c79eff8449b617eef9b3dde249489b107d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T114816692912Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa52e8cc6d6c73011b941e0d79cb6c9bc6938d624fe2cfc573f3e086ca9ea145 +size 5597013 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T114817621398Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T114817621398Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03bb0cf88c80bd92bb73671fe6e8cdae4d5d85ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T114817621398Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322202112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322202112, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322202112, + "device": 5, + "inactive_split_bytes_current": 2703639552, + "inactive_split_bytes_peak": 3871376384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104685633536, + "reserved_bytes_peak": 104685633536, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T11:48:17.621000Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322202112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322202112, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322202112, + "device": 5, + "inactive_split_bytes_current": 2699445248, + "inactive_split_bytes_peak": 3871376384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104685633536, + "reserved_bytes_peak": 104685633536, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T11:48:16.806418Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2703639552, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 3871376384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104685633536, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104685633536, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 606, + "segment_count_current_delta": 0, + "segment_count_peak": 606, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T114817621398Z.pickle", + "timestamp_utc": "2026-08-01T11:48:17.718621Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T114817621398Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T114817621398Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cd473b9fe3cfd633b5deba824e16618f407fe35c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T114817621398Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f28040f8a8e2b504c2ffbc25934b74aba30f569b4cb555427269a80058f580 +size 5598892 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T114820284535Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T114820284535Z.json new file mode 100644 index 0000000000000000000000000000000000000000..64e6f8656cd5eb7f3826dab03dce94d6711332eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T114820284535Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 51981004800, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_peak": 5975309312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.283990Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 5688832, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103322202112, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103322202112, + "device": 5, + "inactive_split_bytes_current": 2703639552, + "inactive_split_bytes_peak": 3871376384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104685633536, + "reserved_bytes_peak": 104685633536, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T11:48:17.729870Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_current_delta": 212725760, + "inactive_split_bytes_peak": 5975309312, + "inactive_split_bytes_peak_delta": 2103932928, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 60687384576, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 60687384576, + "segment_count_current": 959, + "segment_count_current_delta": 353, + "segment_count_peak": 959, + "segment_count_peak_delta": 353, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T114820284535Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.406498Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T114820284535Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T114820284535Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0837cca61e304f0e7ce0abd442407bd9dc85891e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T114820284535Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cc4e0e6e35a268cc9410156dae854a72d95a2d871e7d6432f3e83686d941653 +size 6614160 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T114820969263Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T114820969263Z.json new file mode 100644 index 0000000000000000000000000000000000000000..456924804a0d82aa782923be7629ab273b27bede --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T114820969263Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_peak": 5975309312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.968809Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_peak": 5975309312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.419786Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5975309312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T114820969263Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.976221Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T114820969263Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T114820969263Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e86570b6cb1fa37ae56e91eea17fc2ee6472d1a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T114820969263Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf63dff5acb050133f0447d4bf7c07bfb3a6bdbf2e4796a8a83666de4d0f9465 +size 293758 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T114821516607Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T114821516607Z.json new file mode 100644 index 0000000000000000000000000000000000000000..28e2fa674bf92149d9addf30245213a1ee9d26f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T114821516607Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_peak": 5975309312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:21.516211Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_peak": 5975309312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.977198Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5975309312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T114821516607Z.pickle", + "timestamp_utc": "2026-08-01T11:48:21.523333Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T114821516607Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T114821516607Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d1fd1eb2ed97619b6c1dd09b4214d406f87804a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T114821516607Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec2f05d8669c7e00a2881594fa8d2028c263686bba8fe3a2d6295d162d3392ba +size 293758 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T114822060103Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T114822060103Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7d138caa8ce255cdb75e979eb527593ce138427e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T114822060103Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_peak": 5975309312, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.059704Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_peak": 5975309312, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:21.524318Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5975309312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T114822060103Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.066522Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T114822060103Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T114822060103Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f747a80d4fcb906d81c66338bb6dd0534130d543 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T114822060103Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d1e1fd016d7acaefbd16da0045ce4b0aeebc27ab94af2e206947532afa65530 +size 293758 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T114822604173Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T114822604173Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c9bc589ef8ceddeb4a4d06941c72d16e2efcb88f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T114822604173Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_peak": 5975309312, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.603771Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_peak": 5975309312, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.067484Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5975309312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T114822604173Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.610792Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T114822604173Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T114822604173Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d915ffa01effef2ec4be23b43bfa0ea60d5f323c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T114822604173Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72816f5c0066a7f5c341086aff91ef5cb447ffeb373eaad833d664fc0012fdee +size 293758 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T114823148250Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T114823148250Z.json new file mode 100644 index 0000000000000000000000000000000000000000..91e7d817fb73117d89c47178a528a3a9465b5bf9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T114823148250Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_peak": 5975309312, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:23.147864Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_peak": 5975309312, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.611778Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5975309312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T114823148250Z.pickle", + "timestamp_utc": "2026-08-01T11:48:23.154670Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T114823148250Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T114823148250Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3b3991f6b82fd67b975b08904f618679866416ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T114823148250Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02f59a7ff186758ddf176ff8ecec3d4b45d0c122517f067af2d2480e66b16a68 +size 293758 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T114917758432Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T114917758432Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e635a2a16e02144b68e4f6772db82e4e328d131 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T114917758432Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855801856, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 155303206912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855801856, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66855801856, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2656398336, + "inactive_split_bytes_peak": 5975309312, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 166983630848, + "reserved_bytes_peak": 166983630848, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:49:17.757946Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855801856, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103327890944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155303206912, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103327890944, + "device": 5, + "inactive_split_bytes_current": 2916365312, + "inactive_split_bytes_peak": 5975309312, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:23.155630Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2656398336, + "inactive_split_bytes_current_delta": -259966976, + "inactive_split_bytes_peak": 5975309312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 166983630848, + "reserved_bytes_current_delta": 1610612736, + "reserved_bytes_peak": 166983630848, + "reserved_bytes_peak_delta": 1610612736, + "segment_count_current": 967, + "segment_count_current_delta": 8, + "segment_count_peak": 967, + "segment_count_peak_delta": 8, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T114917758432Z.pickle", + "timestamp_utc": "2026-08-01T11:49:17.843625Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T114917758432Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T114917758432Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f53c97ecc87d06219a189ad528f2e0412d40a685 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T114917758432Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62900b3cd81a6a2bd0570c98cd969a0eeb0c6fb6e629dae5596810ed71af5555 +size 4101078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T114815754645Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T114815754645Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9bb34ffefcb911b0c1d9c9e50d46d2443a80b920 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T114815754645Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 31640019968, + "active_bytes_peak": 79684781568, + "active_bytes_peak_delta": 44218445312, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79684781568, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79684781568, + "device": 6, + "inactive_split_bytes_current": 3156624384, + "inactive_split_bytes_peak": 3694871552, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 96502546432, + "reserved_bytes_peak": 96502546432, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T11:48:15.754009Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 31640019968, + "allocated_bytes_peak": 79684781568, + "allocated_bytes_peak_delta": 44218445312, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:47:06.472888Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3156624384, + "inactive_split_bytes_current_delta": 3023805440, + "inactive_split_bytes_peak": 3694871552, + "inactive_split_bytes_peak_delta": 3534485504, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 96502546432, + "reserved_bytes_current_delta": 57258541056, + "reserved_bytes_peak": 96502546432, + "reserved_bytes_peak_delta": 57258541056, + "segment_count_current": 550, + "segment_count_current_delta": 479, + "segment_count_peak": 550, + "segment_count_peak_delta": 479, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T114815754645Z.pickle", + "timestamp_utc": "2026-08-01T11:48:15.868669Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T114815754645Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T114815754645Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..950ace88b936f79c94b4cc96f2feca760ff7442d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T114815754645Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3e385101fa5c0b4634c81207fdaa97a0d269a19370bc980a7f73580889dff17 +size 5392331 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T114816692809Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T114816692809Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d411dbbc1f841c0a71e2d6112931bcdb4373e987 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T114816692809Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103326366720, + "active_bytes_peak_delta": 23641585152, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103326366720, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103326366720, + "device": 6, + "inactive_split_bytes_current": 2934326272, + "inactive_split_bytes_peak": 3694871552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 105591603200, + "reserved_bytes_peak": 105591603200, + "segment_count_current": 607, + "segment_count_peak": 607, + "timestamp_utc": "2026-08-01T11:48:16.692401Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326366720, + "allocated_bytes_peak_delta": 23641585152, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 79684781568, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 79684781568, + "device": 6, + "inactive_split_bytes_current": 3156624384, + "inactive_split_bytes_peak": 3694871552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 96502546432, + "reserved_bytes_peak": 96502546432, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T11:48:15.887962Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2934326272, + "inactive_split_bytes_current_delta": -222298112, + "inactive_split_bytes_peak": 3694871552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 105591603200, + "reserved_bytes_current_delta": 9089056768, + "reserved_bytes_peak": 105591603200, + "reserved_bytes_peak_delta": 9089056768, + "segment_count_current": 607, + "segment_count_current_delta": 57, + "segment_count_peak": 607, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T114816692809Z.pickle", + "timestamp_utc": "2026-08-01T11:48:16.798900Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T114816692809Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T114816692809Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e038d881e35165199f545378211656fccc44ccf7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T114816692809Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6d1c709fc7c70933cab7566d21a42ae7da53504ec18854a4777c62c5dcac10a +size 5597225 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T114817621277Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T114817621277Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f7cfe02a69d1a1003f8458f1f24f293ea28cc57c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T114817621277Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103326366720, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103326366720, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103326366720, + "device": 6, + "inactive_split_bytes_current": 2936423424, + "inactive_split_bytes_peak": 3698100224, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 105591603200, + "reserved_bytes_peak": 105591603200, + "segment_count_current": 607, + "segment_count_peak": 607, + "timestamp_utc": "2026-08-01T11:48:17.620871Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103326366720, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103326366720, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103326366720, + "device": 6, + "inactive_split_bytes_current": 2934326272, + "inactive_split_bytes_peak": 3694871552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 105591603200, + "reserved_bytes_peak": 105591603200, + "segment_count_current": 607, + "segment_count_peak": 607, + "timestamp_utc": "2026-08-01T11:48:16.814341Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2936423424, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 3698100224, + "inactive_split_bytes_peak_delta": 3228672, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 105591603200, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105591603200, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 607, + "segment_count_current_delta": 0, + "segment_count_peak": 607, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T114817621277Z.pickle", + "timestamp_utc": "2026-08-01T11:48:17.726268Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T114817621277Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T114817621277Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea1c4507d0eb95e1e3800f3a2bc39daef24b40e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T114817621277Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d79255f166ce35d39cea648fde23e15805e44ecdaed91fdc2a6bf909860b2694 +size 5598855 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T114820284486Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T114820284486Z.json new file mode 100644 index 0000000000000000000000000000000000000000..af99f72da0699886e418629a57cf7707f9f5ee1e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T114820284486Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 155305041920, + "active_bytes_peak_delta": 51978675200, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 6201428480, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 165664522240, + "reserved_bytes_peak": 165664522240, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.283916Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103329725952, + "allocated_bytes_peak_delta": 3359232, + "before": { + "active_bytes_current": 66871475200, + "active_bytes_peak": 103326366720, + "allocated_bytes_current": 66871475200, + "allocated_bytes_peak": 103326366720, + "device": 6, + "inactive_split_bytes_current": 2936423424, + "inactive_split_bytes_peak": 3698100224, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 105591603200, + "reserved_bytes_peak": 105591603200, + "segment_count_current": 607, + "segment_count_peak": 607, + "timestamp_utc": "2026-08-01T11:48:17.741083Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_current_delta": -22155264, + "inactive_split_bytes_peak": 6201428480, + "inactive_split_bytes_peak_delta": 2503328256, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 165664522240, + "reserved_bytes_current_delta": 60072919040, + "reserved_bytes_peak": 165664522240, + "reserved_bytes_peak_delta": 60072919040, + "segment_count_current": 959, + "segment_count_current_delta": 352, + "segment_count_peak": 959, + "segment_count_peak_delta": 352, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T114820284486Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.412652Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T114820284486Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T114820284486Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fd307a051ea45004b55462d170e54fec09986395 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T114820284486Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21160a694fa383c88d83b2f37700619ddea51e5a020a288f5aa9b65e64776c59 +size 6614079 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T114820969167Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T114820969167Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f1afd36d432368e5214a6729b5d0da9b45ee706 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T114820969167Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155305041920, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 6201428480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 165664522240, + "reserved_bytes_peak": 165664522240, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.968701Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329725952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 6201428480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 165664522240, + "reserved_bytes_peak": 165664522240, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.429411Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6201428480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 165664522240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165664522240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T114820969167Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.976755Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T114820969167Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T114820969167Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1f994863883cba6f3d3ee0f28e1ba7aaef36abb6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T114820969167Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:026ea4ed402bf7e7b8d34bbf4c95f3294548bb46ef349e45461b160ff6cddca1 +size 294071 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T114821516667Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T114821516667Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fefa4c029c83b01afaee58f13eda2263b3c22269 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T114821516667Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155305041920, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 6201428480, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 165664522240, + "reserved_bytes_peak": 165664522240, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:21.516257Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329725952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 6201428480, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 165664522240, + "reserved_bytes_peak": 165664522240, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.977756Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6201428480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 165664522240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165664522240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T114821516667Z.pickle", + "timestamp_utc": "2026-08-01T11:48:21.525944Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T114821516667Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T114821516667Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a527fd14ac263af314225e8656356266f6d599fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T114821516667Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d485c7c09c23a1e7b555202ef05a40f0719a3bc616e4c54028c2f804d26b009c +size 294071 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T114822060014Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T114822060014Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d4923994f3a05e8e657864366f44586fbe7d585 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T114822060014Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155305041920, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 6201428480, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 165664522240, + "reserved_bytes_peak": 165664522240, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.059618Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329725952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 6201428480, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 165664522240, + "reserved_bytes_peak": 165664522240, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:21.526934Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6201428480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 165664522240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165664522240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T114822060014Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.066784Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T114822060014Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T114822060014Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7564e534eb7252e4a0e2ba914eb07edc7f6ad4d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T114822060014Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e12d55d39b9a726d9ab6b0bddf231fe8f0db16693ee3a44ac9ae16aeac09ba9d +size 294071 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T114822604140Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T114822604140Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fcdfd661ee7bdf8b3d1eb7bb34db7da1e8931ba3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T114822604140Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155305041920, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 6201428480, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 165664522240, + "reserved_bytes_peak": 165664522240, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.603728Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329725952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 6201428480, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 165664522240, + "reserved_bytes_peak": 165664522240, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.067761Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6201428480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 165664522240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165664522240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T114822604140Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.611002Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T114822604140Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T114822604140Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c13eadc7e2d0024b69be663cd460eed977f32970 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T114822604140Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df75d9a546579b9164e39bb20c4649ab301bc60dd5e379df75434891ef5570e7 +size 294071 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T114823148207Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T114823148207Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a8483ca5ad6f24706b2fb362a9f110f92f48e5f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T114823148207Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155305041920, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 6201428480, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 165664522240, + "reserved_bytes_peak": 165664522240, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:23.147806Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103329725952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 6201428480, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 165664522240, + "reserved_bytes_peak": 165664522240, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.611980Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 6201428480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 165664522240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165664522240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T114823148207Z.pickle", + "timestamp_utc": "2026-08-01T11:48:23.154835Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T114823148207Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T114823148207Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c7f9601bb1bf36bf03ebf0ed0273626f8d0f170 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T114823148207Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8171550151f275d759cbfb37452e662baf754de6d511b1881337e50749de9fa +size 294071 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T114917758434Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T114917758434Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f3c2a66e5aec1d8b21f1ed4f8d494c3f960d0bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T114917758434Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855801856, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 155305041920, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855801856, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66855801856, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2656398336, + "inactive_split_bytes_peak": 6201428480, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 166335610880, + "reserved_bytes_peak": 166335610880, + "segment_count_current": 963, + "segment_count_peak": 963, + "timestamp_utc": "2026-08-01T11:49:17.757963Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855801856, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103329725952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436160, + "active_bytes_peak": 155305041920, + "allocated_bytes_current": 66889436160, + "allocated_bytes_peak": 103329725952, + "device": 6, + "inactive_split_bytes_current": 2914268160, + "inactive_split_bytes_peak": 6201428480, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 165664522240, + "reserved_bytes_peak": 165664522240, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:23.155806Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2656398336, + "inactive_split_bytes_current_delta": -257869824, + "inactive_split_bytes_peak": 6201428480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 166335610880, + "reserved_bytes_current_delta": 671088640, + "reserved_bytes_peak": 166335610880, + "reserved_bytes_peak_delta": 671088640, + "segment_count_current": 963, + "segment_count_current_delta": 4, + "segment_count_peak": 963, + "segment_count_peak_delta": 4, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T114917758434Z.pickle", + "timestamp_utc": "2026-08-01T11:49:17.843248Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T114917758434Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T114917758434Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c2e236adbea11211ec088d6a75cd1361b6b25e5c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T114917758434Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4f3ce67e2d86481a1e7d4dc04ad15e2f98e6ca28ccec14e34c995cf74d1e9be +size 4100838 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T114815754491Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T114815754491Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d1ed4bffee04c5c4edc2e2d5e3bf12597bc0d76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T114815754491Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475712, + "active_bytes_current_delta": 31640020480, + "active_bytes_peak": 79684781568, + "active_bytes_peak_delta": 44218445312, + "after": { + "active_bytes_current": 66871475712, + "active_bytes_peak": 79684781568, + "allocated_bytes_current": 66871475712, + "allocated_bytes_peak": 79684781568, + "device": 7, + "inactive_split_bytes_current": 3156623872, + "inactive_split_bytes_peak": 3871212544, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 96267665408, + "reserved_bytes_peak": 96267665408, + "segment_count_current": 552, + "segment_count_peak": 552, + "timestamp_utc": "2026-08-01T11:48:15.754010Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475712, + "allocated_bytes_current_delta": 31640020480, + "allocated_bytes_peak": 79684781568, + "allocated_bytes_peak_delta": 44218445312, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:47:06.472921Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3156623872, + "inactive_split_bytes_current_delta": 3023804928, + "inactive_split_bytes_peak": 3871212544, + "inactive_split_bytes_peak_delta": 3710826496, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 96267665408, + "reserved_bytes_current_delta": 57023660032, + "reserved_bytes_peak": 96267665408, + "reserved_bytes_peak_delta": 57023660032, + "segment_count_current": 552, + "segment_count_current_delta": 481, + "segment_count_peak": 552, + "segment_count_peak_delta": 481, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T114815754491Z.pickle", + "timestamp_utc": "2026-08-01T11:48:15.864733Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T114815754491Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T114815754491Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13ceb7770a5361f267d0acacb395daa9784822e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T114815754491Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:683a59404032cefa32e54fd1ab52b012ae4d6fc7d096a604c796e47c1486bf4d +size 5394239 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T114816692812Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T114816692812Z.json new file mode 100644 index 0000000000000000000000000000000000000000..094beb59627f116bbc2dbab5e3023220826d754e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T114816692812Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322202624, + "active_bytes_peak_delta": 23637421056, + "after": { + "active_bytes_current": 66871475712, + "active_bytes_peak": 103322202624, + "allocated_bytes_current": 66871475712, + "allocated_bytes_peak": 103322202624, + "device": 7, + "inactive_split_bytes_current": 2699444736, + "inactive_split_bytes_peak": 3871212544, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 105155395584, + "reserved_bytes_peak": 105155395584, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T11:48:16.692410Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322202624, + "allocated_bytes_peak_delta": 23637421056, + "before": { + "active_bytes_current": 66871475712, + "active_bytes_peak": 79684781568, + "allocated_bytes_current": 66871475712, + "allocated_bytes_peak": 79684781568, + "device": 7, + "inactive_split_bytes_current": 3156623872, + "inactive_split_bytes_peak": 3871212544, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 96267665408, + "reserved_bytes_peak": 96267665408, + "segment_count_current": 552, + "segment_count_peak": 552, + "timestamp_utc": "2026-08-01T11:48:15.883810Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2699444736, + "inactive_split_bytes_current_delta": -457179136, + "inactive_split_bytes_peak": 3871212544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 105155395584, + "reserved_bytes_current_delta": 8887730176, + "reserved_bytes_peak": 105155395584, + "reserved_bytes_peak_delta": 8887730176, + "segment_count_current": 608, + "segment_count_current_delta": 56, + "segment_count_peak": 608, + "segment_count_peak_delta": 56, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T114816692812Z.pickle", + "timestamp_utc": "2026-08-01T11:48:16.799088Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T114816692812Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T114816692812Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d09eb72101a2e25d3c4b05f69e10068c6c5340c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T114816692812Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7de399f1bd55029330e66288b6c7d9c414ca77cff76b72da36a3f2069a055e4d +size 5599295 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T114817621395Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T114817621395Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3eb275698f76808833d78dcd9c9709344cd3fb9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T114817621395Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871475712, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322202624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871475712, + "active_bytes_peak": 103322202624, + "allocated_bytes_current": 66871475712, + "allocated_bytes_peak": 103322202624, + "device": 7, + "inactive_split_bytes_current": 2703639040, + "inactive_split_bytes_peak": 3871212544, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 105155395584, + "reserved_bytes_peak": 105155395584, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T11:48:17.620958Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871475712, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322202624, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871475712, + "active_bytes_peak": 103322202624, + "allocated_bytes_current": 66871475712, + "allocated_bytes_peak": 103322202624, + "device": 7, + "inactive_split_bytes_current": 2699444736, + "inactive_split_bytes_peak": 3871212544, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 105155395584, + "reserved_bytes_peak": 105155395584, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T11:48:16.814285Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2703639040, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 3871212544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 105155395584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105155395584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 608, + "segment_count_current_delta": 0, + "segment_count_peak": 608, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T114817621395Z.pickle", + "timestamp_utc": "2026-08-01T11:48:17.726367Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T114817621395Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T114817621395Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..adf60e242074f11a3623aeb3ad031c2025be4f6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T114817621395Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1e78d57f3f7b230c2b1aa669335ad6a07043d8a001d5c78ee7a6fc9e1486f3e +size 5601117 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T114820284746Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T114820284746Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f93defb511e4974b3bd865407ff25558680c9ee2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T114820284746Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436672, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 155303207424, + "active_bytes_peak_delta": 51981004800, + "after": { + "active_bytes_current": 66889436672, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66889436672, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_peak": 5937560064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.284073Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436672, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103327891456, + "allocated_bytes_peak_delta": 5688832, + "before": { + "active_bytes_current": 66871475712, + "active_bytes_peak": 103322202624, + "allocated_bytes_current": 66871475712, + "allocated_bytes_peak": 103322202624, + "device": 7, + "inactive_split_bytes_current": 2703639040, + "inactive_split_bytes_peak": 3871212544, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 105155395584, + "reserved_bytes_peak": 105155395584, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T11:48:17.741000Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_current_delta": 208531456, + "inactive_split_bytes_peak": 5937560064, + "inactive_split_bytes_peak_delta": 2066347520, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 60217622528, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 60217622528, + "segment_count_current": 959, + "segment_count_current_delta": 351, + "segment_count_peak": 959, + "segment_count_peak_delta": 351, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T114820284746Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.407906Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T114820284746Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T114820284746Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..557eb3cdf81c7c5406c8f857cfbafdbb23199a59 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T114820284746Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:015c3db7639c239e98eb8c3e5cafe94e20b2951b75c433d22475e63671f228ac +size 6616351 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T114820969285Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T114820969285Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b54c8349be4105f17f57406aced910a23c9b30bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T114820969285Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436672, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303207424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436672, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66889436672, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_peak": 5937560064, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.968791Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436672, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327891456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436672, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66889436672, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_peak": 5937560064, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.424570Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T114820969285Z.pickle", + "timestamp_utc": "2026-08-01T11:48:20.976926Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T114820969285Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T114820969285Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7340fff892b991990d544ec891b28fb52ee47a43 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T114820969285Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eadc663f801b51d26e76bba7a550bf4e80ec85b5f8b52f47f25f8575ae36784a +size 294791 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T114821516598Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T114821516598Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c763b0da11cc7f0b2247db930570dfba97d095f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T114821516598Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436672, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303207424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436672, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66889436672, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_peak": 5937560064, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:21.516200Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436672, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327891456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436672, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66889436672, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_peak": 5937560064, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:20.977995Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T114821516598Z.pickle", + "timestamp_utc": "2026-08-01T11:48:21.523586Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T114821516598Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T114821516598Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e5855b5177f8a45b8f791d341491ba69b72d1642 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T114821516598Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfc022ee3e684806ef4825403477850792753366f85186341d54c3e1129e656a +size 294791 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T114822060042Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T114822060042Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f95fda73c4eeb51430c6d86791781d9688ccd73c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T114822060042Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436672, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303207424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436672, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66889436672, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_peak": 5937560064, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.059658Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436672, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327891456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436672, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66889436672, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_peak": 5937560064, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:21.524536Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T114822060042Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.066606Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T114822060042Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T114822060042Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..241c8e0af9bca08e7518e28eadea52bbe318db1d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T114822060042Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d4f5991efa3b5fe9b32ea9703ebd50234b6735890f8ca005b28f2286bd0dff5 +size 294791 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T114822604195Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T114822604195Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e3519f4a1847fc1d9d1b468d7f20df800036ad04 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T114822604195Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436672, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303207424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436672, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66889436672, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_peak": 5937560064, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.603783Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436672, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327891456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436672, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66889436672, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_peak": 5937560064, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.067541Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T114822604195Z.pickle", + "timestamp_utc": "2026-08-01T11:48:22.611027Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T114822604195Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T114822604195Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f320540ed81915641ce2792fda83d2d018e4770 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T114822604195Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ecad9bad9a9e1de61780f94ecf6accee76a8fcd7f9dd3c3e60d12f629fb7f3d +size 294791 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T114823148296Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T114823148296Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b0a9d29ec5b9c15811e43566582869c4bec4bec0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T114823148296Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889436672, + "active_bytes_current_delta": 0, + "active_bytes_peak": 155303207424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889436672, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66889436672, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_peak": 5937560064, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:23.147884Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889436672, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103327891456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436672, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66889436672, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_peak": 5937560064, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:22.612002Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5937560064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 165373018112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 165373018112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 959, + "segment_count_current_delta": 0, + "segment_count_peak": 959, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T114823148296Z.pickle", + "timestamp_utc": "2026-08-01T11:48:23.155058Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T114823148296Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T114823148296Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..848b9731e3cc76ab7b026b8a9605a016aa351653 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T114823148296Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d693cad5dfe96d4068e8d76460c6bb3226ce00dd6018bcf9d8c3f723bbdc24f +size 294791 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T114917758433Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T114917758433Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0343df485f4474a4975c7bbcab16ffbd1b0916f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T114917758433Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855802368, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 155303207424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855802368, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66855802368, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2656397824, + "inactive_split_bytes_peak": 5937560064, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 166983630848, + "reserved_bytes_peak": 166983630848, + "segment_count_current": 967, + "segment_count_peak": 967, + "timestamp_utc": "2026-08-01T11:49:17.757968Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855802368, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 103327891456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889436672, + "active_bytes_peak": 155303207424, + "allocated_bytes_current": 66889436672, + "allocated_bytes_peak": 103327891456, + "device": 7, + "inactive_split_bytes_current": 2912170496, + "inactive_split_bytes_peak": 5937560064, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 165373018112, + "reserved_bytes_peak": 165373018112, + "segment_count_current": 959, + "segment_count_peak": 959, + "timestamp_utc": "2026-08-01T11:48:23.156012Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2656397824, + "inactive_split_bytes_current_delta": -255772672, + "inactive_split_bytes_peak": 5937560064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 166983630848, + "reserved_bytes_current_delta": 1610612736, + "reserved_bytes_peak": 166983630848, + "reserved_bytes_peak_delta": 1610612736, + "segment_count_current": 967, + "segment_count_current_delta": 8, + "segment_count_peak": 967, + "segment_count_peak_delta": 8, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T114917758433Z.pickle", + "timestamp_utc": "2026-08-01T11:49:17.841956Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T114917758433Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T114917758433Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1092d474a5db4c7434d485a81ed53f9f26b25731 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T114917758433Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01252564026927f898bc9e00f4bfa52fb04ee37a2bfd371cf65537a761f7e6e0 +size 4102546 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..534c99f7aa727e5dbcd0bf954d6dc85d37c70d04 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 2048 +capacity: 1.00 +expert_dispatch_overlap: on +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6003 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 231 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T115216293027Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T115216293027Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee7a4f78232b2220c6c73a544775f394b5c971ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T115216293027Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459529216, + "active_bytes_current_delta": 37228073984, + "active_bytes_peak": 80282818560, + "active_bytes_peak_delta": 44816482304, + "after": { + "active_bytes_current": 72459529216, + "active_bytes_peak": 80282818560, + "allocated_bytes_current": 72459529216, + "allocated_bytes_peak": 79581058560, + "device": 0, + "inactive_split_bytes_current": 3069400064, + "inactive_split_bytes_peak": 5212915200, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 96825507840, + "reserved_bytes_peak": 96825507840, + "segment_count_current": 513, + "segment_count_peak": 513, + "timestamp_utc": "2026-08-01T11:52:16.292473Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459529216, + "allocated_bytes_current_delta": 37228073984, + "allocated_bytes_peak": 79581058560, + "allocated_bytes_peak_delta": 44114722304, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T11:51:07.374794Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3069400064, + "inactive_split_bytes_current_delta": 2934483968, + "inactive_split_bytes_peak": 5212915200, + "inactive_split_bytes_peak_delta": 5052283392, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 96825507840, + "reserved_bytes_current_delta": 57579405312, + "reserved_bytes_peak": 96825507840, + "reserved_bytes_peak_delta": 57579405312, + "segment_count_current": 513, + "segment_count_current_delta": 441, + "segment_count_peak": 513, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T115216293027Z.pickle", + "timestamp_utc": "2026-08-01T11:52:16.410332Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T115216293027Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T115216293027Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f7ecdeb02c99596c57c592856083b0da07d12e23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T115216293027Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eb340e98d46a326c6ae14cce30a01b3ee95fc078ee26deaf0cd2b5be8729a88 +size 5957062 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T115217414851Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T115217414851Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1e69511b3c58ea5d57af1fd5ed027771a8a331d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T115217414851Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459529216, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103573830656, + "active_bytes_peak_delta": 23291012096, + "after": { + "active_bytes_current": 72459529216, + "active_bytes_peak": 103573830656, + "allocated_bytes_current": 72459529216, + "allocated_bytes_peak": 103221509120, + "device": 0, + "inactive_split_bytes_current": 3543356416, + "inactive_split_bytes_peak": 5212915200, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106390618112, + "reserved_bytes_peak": 106390618112, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T11:52:17.414447Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459529216, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103221509120, + "allocated_bytes_peak_delta": 23640450560, + "before": { + "active_bytes_current": 72459529216, + "active_bytes_peak": 80282818560, + "allocated_bytes_current": 72459529216, + "allocated_bytes_peak": 79581058560, + "device": 0, + "inactive_split_bytes_current": 3069400064, + "inactive_split_bytes_peak": 5212915200, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 96825507840, + "reserved_bytes_peak": 96825507840, + "segment_count_current": 513, + "segment_count_peak": 513, + "timestamp_utc": "2026-08-01T11:52:16.429236Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3543356416, + "inactive_split_bytes_current_delta": 473956352, + "inactive_split_bytes_peak": 5212915200, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106390618112, + "reserved_bytes_current_delta": 9565110272, + "reserved_bytes_peak": 106390618112, + "reserved_bytes_peak_delta": 9565110272, + "segment_count_current": 581, + "segment_count_current_delta": 68, + "segment_count_peak": 581, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T115217414851Z.pickle", + "timestamp_utc": "2026-08-01T11:52:17.523531Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T115217414851Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T115217414851Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0f07d8f3c260a8f17cf4ad5bedd45246b243be1a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T115217414851Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b171c2ae52a216547138d43f034b51ff48edf8297c5d1f7085e5c937e71b9e8d +size 6618274 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T115218477799Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T115218477799Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd89dcd9d4129833806126c0d9ea905f5ed8a25a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T115218477799Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459529216, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103573830656, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72459529216, + "active_bytes_peak": 103573830656, + "allocated_bytes_current": 72459529216, + "allocated_bytes_peak": 103221509120, + "device": 0, + "inactive_split_bytes_current": 3547550720, + "inactive_split_bytes_peak": 5212915200, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106390618112, + "reserved_bytes_peak": 106390618112, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T11:52:18.477406Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459529216, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103221509120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72459529216, + "active_bytes_peak": 103573830656, + "allocated_bytes_current": 72459529216, + "allocated_bytes_peak": 103221509120, + "device": 0, + "inactive_split_bytes_current": 3543356416, + "inactive_split_bytes_peak": 5212915200, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106390618112, + "reserved_bytes_peak": 106390618112, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T11:52:17.538088Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3547550720, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 5212915200, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106390618112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106390618112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 581, + "segment_count_current_delta": 0, + "segment_count_peak": 581, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T115218477799Z.pickle", + "timestamp_utc": "2026-08-01T11:52:18.588295Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T115218477799Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T115218477799Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e3f529ce07aed74e62d81ffbae6d85b1949d9625 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T115218477799Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc1f29d1dbf80569fc2599bde5f008330b6ade050646f66511d17a18bfb929c3 +size 6626874 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T115221500037Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T115221500037Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fac87d11ca8c211ba0a7d05bb837553462c50d98 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T115221500037Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72477490176, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 160835252224, + "active_bytes_peak_delta": 57261421568, + "after": { + "active_bytes_current": 72477490176, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 72477490176, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_peak": 11123106816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_peak": 175470804992, + "segment_count_current": 915, + "segment_count_peak": 915, + "timestamp_utc": "2026-08-01T11:52:21.499503Z", + "world_size": 8 + }, + "allocated_bytes_current": 72477490176, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103223184896, + "allocated_bytes_peak_delta": 1675776, + "before": { + "active_bytes_current": 72459529216, + "active_bytes_peak": 103573830656, + "allocated_bytes_current": 72459529216, + "allocated_bytes_peak": 103221509120, + "device": 0, + "inactive_split_bytes_current": 3547550720, + "inactive_split_bytes_peak": 5212915200, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106390618112, + "reserved_bytes_peak": 106390618112, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T11:52:18.602382Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_current_delta": 4146982912, + "inactive_split_bytes_peak": 11123106816, + "inactive_split_bytes_peak_delta": 5910191616, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_current_delta": 69080186880, + "reserved_bytes_peak": 175470804992, + "reserved_bytes_peak_delta": 69080186880, + "segment_count_current": 915, + "segment_count_current_delta": 334, + "segment_count_peak": 915, + "segment_count_peak_delta": 334, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T115221500037Z.pickle", + "timestamp_utc": "2026-08-01T11:52:21.631911Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T115221500037Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T115221500037Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..906f5da8323691b2e3a4bf2b807fcc35ba04fbee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T115221500037Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:384ec82e410d1bea584b4ad8eb84b4f6ddcb5d9995f4c5357dac2d5336f680ea +size 6956027 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T115222449263Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T115222449263Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ea7167d400b9f2fb7e24f1ea950461204bb6b3b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T115222449263Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72477490176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160835252224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72477490176, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 72477490176, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_peak": 11123106816, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_peak": 175470804992, + "segment_count_current": 915, + "segment_count_peak": 915, + "timestamp_utc": "2026-08-01T11:52:22.448768Z", + "world_size": 8 + }, + "allocated_bytes_current": 72477490176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223184896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72477490176, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 72477490176, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_peak": 11123106816, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_peak": 175470804992, + "segment_count_current": 915, + "segment_count_peak": 915, + "timestamp_utc": "2026-08-01T11:52:21.650257Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11123106816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175470804992, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 915, + "segment_count_current_delta": 0, + "segment_count_peak": 915, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T115222449263Z.pickle", + "timestamp_utc": "2026-08-01T11:52:22.456220Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T115222449263Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T115222449263Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..86c29b30a45637da61e38bc4a1d54a4cc464c91b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T115222449263Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c7480c63cf8898f91fb60a10619a158be4833de7327d3a5b967e07981b2971a +size 250944 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T115223256657Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T115223256657Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef8276d42b810f54d743ba230fe017a60f7ee48f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T115223256657Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72477490176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160835252224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72477490176, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 72477490176, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_peak": 11123106816, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_peak": 175470804992, + "segment_count_current": 915, + "segment_count_peak": 915, + "timestamp_utc": "2026-08-01T11:52:23.256220Z", + "world_size": 8 + }, + "allocated_bytes_current": 72477490176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223184896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72477490176, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 72477490176, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_peak": 11123106816, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_peak": 175470804992, + "segment_count_current": 915, + "segment_count_peak": 915, + "timestamp_utc": "2026-08-01T11:52:22.457274Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11123106816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175470804992, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 915, + "segment_count_current_delta": 0, + "segment_count_peak": 915, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T115223256657Z.pickle", + "timestamp_utc": "2026-08-01T11:52:23.263438Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T115223256657Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T115223256657Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f178335703e387d90db5ceb3b15dcf003f4713b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T115223256657Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53b48003f90557e4431a324a2d72a40fa47213106e303c121c07fc5bec018a8e +size 250944 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T115224063251Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T115224063251Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b2fd3344c4969d5943fea3a5ff38027049a202b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T115224063251Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72477490176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160835252224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72477490176, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 72477490176, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_peak": 11123106816, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_peak": 175470804992, + "segment_count_current": 915, + "segment_count_peak": 915, + "timestamp_utc": "2026-08-01T11:52:24.062848Z", + "world_size": 8 + }, + "allocated_bytes_current": 72477490176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223184896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72477490176, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 72477490176, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_peak": 11123106816, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_peak": 175470804992, + "segment_count_current": 915, + "segment_count_peak": 915, + "timestamp_utc": "2026-08-01T11:52:23.264416Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11123106816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175470804992, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 915, + "segment_count_current_delta": 0, + "segment_count_peak": 915, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T115224063251Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.069329Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T115224063251Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T115224063251Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..10ec90ee922b6338d0f5245f11bd3f1a2a511bd4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T115224063251Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5605fc969c4eb82bd16f717862452487d8fc84471b1e17c594cd88922c131fd +size 250944 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T115224870319Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T115224870319Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d044246740e63831e80aed4db92c498b70bb6c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T115224870319Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72477490176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160835252224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72477490176, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 72477490176, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_peak": 11123106816, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_peak": 175470804992, + "segment_count_current": 915, + "segment_count_peak": 915, + "timestamp_utc": "2026-08-01T11:52:24.869797Z", + "world_size": 8 + }, + "allocated_bytes_current": 72477490176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223184896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72477490176, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 72477490176, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_peak": 11123106816, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_peak": 175470804992, + "segment_count_current": 915, + "segment_count_peak": 915, + "timestamp_utc": "2026-08-01T11:52:24.070278Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11123106816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175470804992, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 915, + "segment_count_current_delta": 0, + "segment_count_peak": 915, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T115224870319Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.877152Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T115224870319Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T115224870319Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03c7ad2c0fa048036d695c9e1a727c3ddb992958 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T115224870319Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcb1bdc71957e93561325012206249fbebaadd518e225afc0d157ab44a81e0ee +size 250944 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T115225678967Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T115225678967Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4af58ca6246d9f78a4ce2601e1e2cb998ec5d79f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T115225678967Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72477490176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160835252224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72477490176, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 72477490176, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_peak": 11123106816, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_peak": 175470804992, + "segment_count_current": 915, + "segment_count_peak": 915, + "timestamp_utc": "2026-08-01T11:52:25.678560Z", + "world_size": 8 + }, + "allocated_bytes_current": 72477490176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223184896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72477490176, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 72477490176, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_peak": 11123106816, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_peak": 175470804992, + "segment_count_current": 915, + "segment_count_peak": 915, + "timestamp_utc": "2026-08-01T11:52:24.878154Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11123106816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175470804992, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 915, + "segment_count_current_delta": 0, + "segment_count_peak": 915, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T115225678967Z.pickle", + "timestamp_utc": "2026-08-01T11:52:25.684867Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T115225678967Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T115225678967Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1298d78c65568dc83d53d78fcb316fb48790eb5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T115225678967Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e719b21506cf48e9d5d8a944b57496f85cedd464cb2d7f93539f56891fd3740 +size 250944 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T115314466687Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T115314466687Z.json new file mode 100644 index 0000000000000000000000000000000000000000..60821995cd5fcd6146159f8a34a550bbf8d14d8f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T115314466687Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61220209664, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 160835252224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61220209664, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 61220209664, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 3707616256, + "inactive_split_bytes_peak": 11123106816, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 176458563584, + "reserved_bytes_peak": 176458563584, + "segment_count_current": 924, + "segment_count_peak": 924, + "timestamp_utc": "2026-08-01T11:53:14.466233Z", + "world_size": 8 + }, + "allocated_bytes_current": 61220209664, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103223184896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72477490176, + "active_bytes_peak": 160835252224, + "allocated_bytes_current": 72477490176, + "allocated_bytes_peak": 103223184896, + "device": 0, + "inactive_split_bytes_current": 7694533632, + "inactive_split_bytes_peak": 11123106816, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 175470804992, + "reserved_bytes_peak": 175470804992, + "segment_count_current": 915, + "segment_count_peak": 915, + "timestamp_utc": "2026-08-01T11:52:25.685819Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3707616256, + "inactive_split_bytes_current_delta": -3986917376, + "inactive_split_bytes_peak": 11123106816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 176458563584, + "reserved_bytes_current_delta": 987758592, + "reserved_bytes_peak": 176458563584, + "reserved_bytes_peak_delta": 987758592, + "segment_count_current": 924, + "segment_count_current_delta": 9, + "segment_count_peak": 924, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T115314466687Z.pickle", + "timestamp_utc": "2026-08-01T11:53:14.552916Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T115314466687Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T115314466687Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c7d94fe93037eb9a2d97c40e37a263d02ba72c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T115314466687Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0465ba71711bd420afc53b46a19a455a377d592cb7b9ca8d4341169043637386 +size 4097441 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T115216293043Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T115216293043Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4806a8a077b7fea281dc2262d61ade83c7cc2825 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T115216293043Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 37226521600, + "active_bytes_peak": 80282927104, + "active_bytes_peak_delta": 44816590848, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 80282927104, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 79582477824, + "device": 1, + "inactive_split_bytes_current": 2926248960, + "inactive_split_bytes_peak": 5561548288, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 96588529664, + "reserved_bytes_peak": 96588529664, + "segment_count_current": 508, + "segment_count_peak": 508, + "timestamp_utc": "2026-08-01T11:52:16.292444Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 37226521600, + "allocated_bytes_peak": 79582477824, + "allocated_bytes_peak_delta": 44116141568, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:51:07.374873Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2926248960, + "inactive_split_bytes_current_delta": 2793430016, + "inactive_split_bytes_peak": 5561548288, + "inactive_split_bytes_peak_delta": 5401162240, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 96588529664, + "reserved_bytes_current_delta": 57344524288, + "reserved_bytes_peak": 96588529664, + "reserved_bytes_peak_delta": 57344524288, + "segment_count_current": 508, + "segment_count_current_delta": 437, + "segment_count_peak": 508, + "segment_count_peak_delta": 437, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T115216293043Z.pickle", + "timestamp_utc": "2026-08-01T11:52:16.407537Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T115216293043Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T115216293043Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7b9ca2affb8fde0dd4a8620ae49e160e03f740f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T115216293043Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ead434acea4dabf78c425962994bf998636bbe701b0238e83f48cffebc50516 +size 5947887 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T115217414852Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T115217414852Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bdfc62e8fb5248ac63bf75499d7237a890054d70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T115217414852Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103575109632, + "active_bytes_peak_delta": 23292182528, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575109632, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103222788096, + "device": 1, + "inactive_split_bytes_current": 3513451520, + "inactive_split_bytes_peak": 5561548288, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T11:52:17.414447Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103222788096, + "allocated_bytes_peak_delta": 23640310272, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 80282927104, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 79582477824, + "device": 1, + "inactive_split_bytes_current": 2926248960, + "inactive_split_bytes_peak": 5561548288, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 96588529664, + "reserved_bytes_peak": 96588529664, + "segment_count_current": 508, + "segment_count_peak": 508, + "timestamp_utc": "2026-08-01T11:52:16.424703Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3513451520, + "inactive_split_bytes_current_delta": 587202560, + "inactive_split_bytes_peak": 5561548288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106380132352, + "reserved_bytes_current_delta": 9791602688, + "reserved_bytes_peak": 106380132352, + "reserved_bytes_peak_delta": 9791602688, + "segment_count_current": 577, + "segment_count_current_delta": 69, + "segment_count_peak": 577, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T115217414852Z.pickle", + "timestamp_utc": "2026-08-01T11:52:17.520364Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T115217414852Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T115217414852Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..223000290cd7040c2b0cefa78b0eaac446c479ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T115217414852Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2836a5c7e041536846abe3652c95efaa21af1f5d52a74d90952ec61215bdb513 +size 5941376 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T115218477816Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T115218477816Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea76f787a2448cdbbbfa06dce1134d2de06121bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T115218477816Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103575109632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575109632, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103222788096, + "device": 1, + "inactive_split_bytes_current": 3926590464, + "inactive_split_bytes_peak": 5561548288, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T11:52:18.477425Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103222788096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575109632, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103222788096, + "device": 1, + "inactive_split_bytes_current": 3513451520, + "inactive_split_bytes_peak": 5561548288, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T11:52:17.533803Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3926590464, + "inactive_split_bytes_current_delta": 413138944, + "inactive_split_bytes_peak": 5561548288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106380132352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106380132352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 577, + "segment_count_current_delta": 0, + "segment_count_peak": 577, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T115218477816Z.pickle", + "timestamp_utc": "2026-08-01T11:52:18.583667Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T115218477816Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T115218477816Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1927bbaf4ce3c1471ae01d9e8475896b8c3727d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T115218477816Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:954f8508b91bb22557380132d99e221dd05686d8d7e1352a82d001a6e8af247f +size 6068766 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T115221500042Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T115221500042Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5ccf859bc5fc2e089cccbabe6f6fdc4431c4c32 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T115221500042Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 160843661312, + "active_bytes_peak_delta": 57268551680, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_peak": 11864953856, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 175789572096, + "reserved_bytes_peak": 175789572096, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:21.499480Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103229890560, + "allocated_bytes_peak_delta": 7102464, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575109632, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103222788096, + "device": 1, + "inactive_split_bytes_current": 3926590464, + "inactive_split_bytes_peak": 5561548288, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T11:52:18.596741Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_current_delta": 4157468672, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 6303405568, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 175789572096, + "reserved_bytes_current_delta": 69409439744, + "reserved_bytes_peak": 175789572096, + "reserved_bytes_peak_delta": 69409439744, + "segment_count_current": 905, + "segment_count_current_delta": 328, + "segment_count_peak": 905, + "segment_count_peak_delta": 328, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T115221500042Z.pickle", + "timestamp_utc": "2026-08-01T11:52:21.623125Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T115221500042Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T115221500042Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..780e9e36ed8c28877f4cf1fecd286ab830f4f941 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T115221500042Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c7ba64d58aa1e8d2da45d7e6e183fd72cb33a84e1477c2dbe83267b83925ca3 +size 6955695 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T115222449265Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T115222449265Z.json new file mode 100644 index 0000000000000000000000000000000000000000..86ba376692091934287ef98b33edf705f18a425d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T115222449265Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160843661312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_peak": 11864953856, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 175789572096, + "reserved_bytes_peak": 175789572096, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:22.448766Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103229890560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_peak": 11864953856, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 175789572096, + "reserved_bytes_peak": 175789572096, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:21.638332Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 175789572096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175789572096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T115222449265Z.pickle", + "timestamp_utc": "2026-08-01T11:52:22.455749Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T115222449265Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T115222449265Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6526e23c1973918ea45ec63f2c66c5d71fa4db3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T115222449265Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f81e7843a5fabc8e90df9b4f0334e6094292696c1c17a3b7acc03f130cfc2a3 +size 248676 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T115223256669Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T115223256669Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9871e8b3169dc16d46306cd5c80f81a47cc9a92a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T115223256669Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160843661312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_peak": 11864953856, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 175789572096, + "reserved_bytes_peak": 175789572096, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:23.256252Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103229890560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_peak": 11864953856, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 175789572096, + "reserved_bytes_peak": 175789572096, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:22.456695Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 175789572096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175789572096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T115223256669Z.pickle", + "timestamp_utc": "2026-08-01T11:52:23.263026Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T115223256669Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T115223256669Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dbef17e6d9b669458a1b6063be49654f35b3e855 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T115223256669Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73c56be06eabcb46e1f33a7afc81095639c92c3a1d5503d029b35b089fa7ced1 +size 248676 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T115224063266Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T115224063266Z.json new file mode 100644 index 0000000000000000000000000000000000000000..abbe5c21b65401805f082f8fdb06245057d74228 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T115224063266Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160843661312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_peak": 11864953856, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 175789572096, + "reserved_bytes_peak": 175789572096, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:24.062871Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103229890560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_peak": 11864953856, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 175789572096, + "reserved_bytes_peak": 175789572096, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:23.263994Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 175789572096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175789572096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T115224063266Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.069101Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T115224063266Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T115224063266Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..75d245a576ff473be3f99e24d3c03e1d1ab588e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T115224063266Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60acb5368fd788d38c2a80722106ce02f66f7a0c49085ef8bf91365007161c87 +size 248676 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T115224870091Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T115224870091Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3256d317ddddec4183d707ca892c26baffe729f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T115224870091Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160843661312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_peak": 11864953856, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 175789572096, + "reserved_bytes_peak": 175789572096, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:24.869684Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103229890560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_peak": 11864953856, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 175789572096, + "reserved_bytes_peak": 175789572096, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:24.070023Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 175789572096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175789572096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T115224870091Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.876013Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T115224870091Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T115224870091Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f1cda12903ace1074b6edfef63f9e8f9dc64afb4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T115224870091Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55058a457a31b9cf0abedd99bd0c756d2d9b10006455088aa39a90735f433866 +size 248676 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T115225679033Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T115225679033Z.json new file mode 100644 index 0000000000000000000000000000000000000000..99d00102d5414ed6ae60e74b45da16aa5968e63b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T115225679033Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160843661312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_peak": 11864953856, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 175789572096, + "reserved_bytes_peak": 175789572096, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:25.678632Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103229890560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_peak": 11864953856, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 175789572096, + "reserved_bytes_peak": 175789572096, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:24.876963Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 175789572096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175789572096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T115225679033Z.pickle", + "timestamp_utc": "2026-08-01T11:52:25.685372Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T115225679033Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T115225679033Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6bed3416ca4bed4ff033df74ebdcab07a2ac3b0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T115225679033Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddb116ced9d32b5879c5299aac3c93dabd75a79467a05607153da0a3325c0d4b +size 248676 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T115314466638Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T115314466638Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a168af4680d7b337fd026147066e5df2fa9374d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T115314466638Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61218657280, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 160843661312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61218657280, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 61218657280, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 4214582272, + "inactive_split_bytes_peak": 11864953856, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 176542449664, + "reserved_bytes_peak": 176542449664, + "segment_count_current": 913, + "segment_count_peak": 913, + "timestamp_utc": "2026-08-01T11:53:14.466159Z", + "world_size": 8 + }, + "allocated_bytes_current": 61218657280, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103229890560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160843661312, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103229890560, + "device": 1, + "inactive_split_bytes_current": 8084059136, + "inactive_split_bytes_peak": 11864953856, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 175789572096, + "reserved_bytes_peak": 175789572096, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:25.686291Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4214582272, + "inactive_split_bytes_current_delta": -3869476864, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 176542449664, + "reserved_bytes_current_delta": 752877568, + "reserved_bytes_peak": 176542449664, + "reserved_bytes_peak_delta": 752877568, + "segment_count_current": 913, + "segment_count_current_delta": 8, + "segment_count_peak": 913, + "segment_count_peak_delta": 8, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T115314466638Z.pickle", + "timestamp_utc": "2026-08-01T11:53:14.551284Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T115314466638Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T115314466638Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a46be32d454320148d446809c7641f35954e1492 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T115314466638Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b591d39e59a7011da8fdb4493bd8f8dc010429560ca39f539da5a0edc31968f +size 4059290 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T115216295186Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T115216295186Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aac35412c1a59e2f6207f4ffaeebc44e4058ba69 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T115216295186Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 37226521600, + "active_bytes_peak": 79934799360, + "active_bytes_peak_delta": 44468463104, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 79934799360, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 79582477824, + "device": 2, + "inactive_split_bytes_current": 2926248960, + "inactive_split_bytes_peak": 5561548288, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 96588529664, + "reserved_bytes_peak": 96588529664, + "segment_count_current": 508, + "segment_count_peak": 508, + "timestamp_utc": "2026-08-01T11:52:16.294541Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 37226521600, + "allocated_bytes_peak": 79582477824, + "allocated_bytes_peak_delta": 44116141568, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:51:07.374651Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2926248960, + "inactive_split_bytes_current_delta": 2793430016, + "inactive_split_bytes_peak": 5561548288, + "inactive_split_bytes_peak_delta": 5401162240, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 96588529664, + "reserved_bytes_current_delta": 57344524288, + "reserved_bytes_peak": 96588529664, + "reserved_bytes_peak_delta": 57344524288, + "segment_count_current": 508, + "segment_count_current_delta": 437, + "segment_count_peak": 508, + "segment_count_peak_delta": 437, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T115216295186Z.pickle", + "timestamp_utc": "2026-08-01T11:52:16.410618Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T115216295186Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T115216295186Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..85a0740c503a21160eccedcd9e013a41b4da8219 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T115216295186Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f517ceece74aef8cc5c9953d64d0917a2fcf567f6a585ab17c7658849747fedc +size 5948033 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T115217414880Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T115217414880Z.json new file mode 100644 index 0000000000000000000000000000000000000000..855bb56d9746b4f287229b16987000e7b3dea651 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T115217414880Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103575109632, + "active_bytes_peak_delta": 23640310272, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575109632, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103222788096, + "device": 2, + "inactive_split_bytes_current": 4161471488, + "inactive_split_bytes_peak": 5561548288, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T11:52:17.414466Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103222788096, + "allocated_bytes_peak_delta": 23640310272, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 79934799360, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 79582477824, + "device": 2, + "inactive_split_bytes_current": 2926248960, + "inactive_split_bytes_peak": 5561548288, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 96588529664, + "reserved_bytes_peak": 96588529664, + "segment_count_current": 508, + "segment_count_peak": 508, + "timestamp_utc": "2026-08-01T11:52:16.431026Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4161471488, + "inactive_split_bytes_current_delta": 1235222528, + "inactive_split_bytes_peak": 5561548288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106380132352, + "reserved_bytes_current_delta": 9791602688, + "reserved_bytes_peak": 106380132352, + "reserved_bytes_peak_delta": 9791602688, + "segment_count_current": 577, + "segment_count_current_delta": 69, + "segment_count_peak": 577, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T115217414880Z.pickle", + "timestamp_utc": "2026-08-01T11:52:17.523349Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T115217414880Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T115217414880Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8af918c843b97738798d64a4f763ff256bd308ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T115217414880Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a979ae6765c5c3dc3202ff08550e63254cb6c7b184e967bfa573d0a2af87fdaa +size 5940364 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T115218477840Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T115218477840Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c0b8f95b628b21ec0cd6c4c406ed6dbec82457c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T115218477840Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103575109632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575109632, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103222788096, + "device": 2, + "inactive_split_bytes_current": 3691709440, + "inactive_split_bytes_peak": 5561548288, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T11:52:18.477450Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103222788096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575109632, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103222788096, + "device": 2, + "inactive_split_bytes_current": 4161471488, + "inactive_split_bytes_peak": 5561548288, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T11:52:17.535922Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3691709440, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 5561548288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106380132352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106380132352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 577, + "segment_count_current_delta": 0, + "segment_count_peak": 577, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T115218477840Z.pickle", + "timestamp_utc": "2026-08-01T11:52:18.577824Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T115218477840Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T115218477840Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..90b0d6dbe4507b56336587769226c2ad63e42552 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T115218477840Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0afa91a38b5f9e5617808babd202446f2b9291d18d81e50fcf5794aa70293039 +size 6065682 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T115221499915Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T115221499915Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2886a429bc333133fd5bc21358838afe2c8ff70e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T115221499915Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 160842088448, + "active_bytes_peak_delta": 57266978816, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_peak": 11864953856, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 175672131584, + "reserved_bytes_peak": 175672131584, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:21.499410Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103228317696, + "allocated_bytes_peak_delta": 5529600, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575109632, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103222788096, + "device": 2, + "inactive_split_bytes_current": 3691709440, + "inactive_split_bytes_peak": 5561548288, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T11:52:18.589482Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_current_delta": 4509790208, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 6303405568, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 175672131584, + "reserved_bytes_current_delta": 69291999232, + "reserved_bytes_peak": 175672131584, + "reserved_bytes_peak_delta": 69291999232, + "segment_count_current": 905, + "segment_count_current_delta": 328, + "segment_count_peak": 905, + "segment_count_peak_delta": 328, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T115221499915Z.pickle", + "timestamp_utc": "2026-08-01T11:52:21.623782Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T115221499915Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T115221499915Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b0b4b51cd02c11d2a82e4051a2caa5a20a496fd4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T115221499915Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:954e911a1f9ca4910516b814979259c5f253e837474fd69bb6181e0427f36119 +size 6955644 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T115222449251Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T115222449251Z.json new file mode 100644 index 0000000000000000000000000000000000000000..83b6b0dddd0e37edc3d85280ab7e7c8ced9e23a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T115222449251Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160842088448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_peak": 11864953856, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 175672131584, + "reserved_bytes_peak": 175672131584, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:22.448777Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103228317696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_peak": 11864953856, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 175672131584, + "reserved_bytes_peak": 175672131584, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:21.639983Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 175672131584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175672131584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T115222449251Z.pickle", + "timestamp_utc": "2026-08-01T11:52:22.455589Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T115222449251Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T115222449251Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..50d37f62f2ae12ee51595b08e27b9526f315fd25 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T115222449251Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abadd405fad14c0bec111408efc89c324cbfa203795807169ee37b47c1dee78a +size 248670 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T115223256624Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T115223256624Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0448069e93bc1cebc37a30be8560be0a6d7a6728 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T115223256624Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160842088448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_peak": 11864953856, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 175672131584, + "reserved_bytes_peak": 175672131584, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:23.256233Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103228317696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_peak": 11864953856, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 175672131584, + "reserved_bytes_peak": 175672131584, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:22.456523Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 175672131584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175672131584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T115223256624Z.pickle", + "timestamp_utc": "2026-08-01T11:52:23.262465Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T115223256624Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T115223256624Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f9b1b2a132a6191d11bcacc6d575f4d25fd40ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T115223256624Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c646e8672e10c2b384418a645b0e03cc635752b6bbb0ebfa38a72dc2d0175db +size 248670 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T115224063241Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T115224063241Z.json new file mode 100644 index 0000000000000000000000000000000000000000..112d88b30016c9955f9f85e018e5833b9ba21ecf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T115224063241Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160842088448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_peak": 11864953856, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 175672131584, + "reserved_bytes_peak": 175672131584, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:24.062845Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103228317696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_peak": 11864953856, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 175672131584, + "reserved_bytes_peak": 175672131584, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:23.263374Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 175672131584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175672131584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T115224063241Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.068844Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T115224063241Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T115224063241Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..06403565a3980c21e0f08c9f8aad4f3365f1a168 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T115224063241Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee1a6f544d7fc7cfa1c98c40eb64dfcefdcaf143ab1f939b80c9bf839b5b8459 +size 248670 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T115224870096Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T115224870096Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f12d377d26fa0547497a92a72ec558905f98ca73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T115224870096Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160842088448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_peak": 11864953856, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 175672131584, + "reserved_bytes_peak": 175672131584, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:24.869703Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103228317696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_peak": 11864953856, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 175672131584, + "reserved_bytes_peak": 175672131584, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:24.069742Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 175672131584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175672131584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T115224870096Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.875960Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T115224870096Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T115224870096Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..77170a9e680a4a8dbbb31969d280ef1d3c97598c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T115224870096Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4573c1c356dbe139d6fad93031ab5e1f299f77e88135fae781d19043348c1b2 +size 248670 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T115225678962Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T115225678962Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9cc8fc18873550e824ac4820aca1031c582b143e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T115225678962Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160842088448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_peak": 11864953856, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 175672131584, + "reserved_bytes_peak": 175672131584, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:25.678572Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103228317696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_peak": 11864953856, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 175672131584, + "reserved_bytes_peak": 175672131584, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:24.876899Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 175672131584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175672131584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T115225678962Z.pickle", + "timestamp_utc": "2026-08-01T11:52:25.684523Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T115225678962Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T115225678962Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cafcbf0720d27900e0d2668f109a65e91a9e462d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T115225678962Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d31f08b3f73f42fc7cae80540499a617fc2c0a323007ac30fc2dce4ed27dfe6 +size 248670 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T115314466637Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T115314466637Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca5fa92a1c98b0048b5abd6f89250056e177a8a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T115314466637Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61218657280, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 160842088448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61218657280, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 61218657280, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 4214582272, + "inactive_split_bytes_peak": 11864953856, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 176659890176, + "reserved_bytes_peak": 176659890176, + "segment_count_current": 914, + "segment_count_peak": 914, + "timestamp_utc": "2026-08-01T11:53:14.466155Z", + "world_size": 8 + }, + "allocated_bytes_current": 61218657280, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103228317696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160842088448, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103228317696, + "device": 2, + "inactive_split_bytes_current": 8201499648, + "inactive_split_bytes_peak": 11864953856, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 175672131584, + "reserved_bytes_peak": 175672131584, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:25.685421Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4214582272, + "inactive_split_bytes_current_delta": -3986917376, + "inactive_split_bytes_peak": 11864953856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 176659890176, + "reserved_bytes_current_delta": 987758592, + "reserved_bytes_peak": 176659890176, + "reserved_bytes_peak_delta": 987758592, + "segment_count_current": 914, + "segment_count_current_delta": 9, + "segment_count_peak": 914, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T115314466637Z.pickle", + "timestamp_utc": "2026-08-01T11:53:14.550857Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T115314466637Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T115314466637Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1172dc48543f0e5bf54205c9d074943325240f8b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T115314466637Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b8e143d596bdba564a72cab385d9dfc5434f599c5c35fce02e3e5fcdab7a586 +size 4059567 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T115216293206Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T115216293206Z.json new file mode 100644 index 0000000000000000000000000000000000000000..769cbad33ccd45d725e704405897a8dac253cea7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T115216293206Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 37226521600, + "active_bytes_peak": 79930605056, + "active_bytes_peak_delta": 44464268800, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 79930605056, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 79578283520, + "device": 3, + "inactive_split_bytes_current": 2691367936, + "inactive_split_bytes_peak": 5561548288, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 96118767616, + "reserved_bytes_peak": 96118767616, + "segment_count_current": 509, + "segment_count_peak": 509, + "timestamp_utc": "2026-08-01T11:52:16.292587Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 37226521600, + "allocated_bytes_peak": 79578283520, + "allocated_bytes_peak_delta": 44111947264, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:51:07.374655Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2691367936, + "inactive_split_bytes_current_delta": 2558548992, + "inactive_split_bytes_peak": 5561548288, + "inactive_split_bytes_peak_delta": 5401162240, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 96118767616, + "reserved_bytes_current_delta": 56874762240, + "reserved_bytes_peak": 96118767616, + "reserved_bytes_peak_delta": 56874762240, + "segment_count_current": 509, + "segment_count_current_delta": 438, + "segment_count_peak": 509, + "segment_count_peak_delta": 438, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T115216293206Z.pickle", + "timestamp_utc": "2026-08-01T11:52:16.416299Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T115216293206Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T115216293206Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8d9025fe931e54eceec323b6e0b70c7baa82837f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T115216293206Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d112fbef559915fbdb61279cfc2e7351c530591aacd4e90f80f1d0914ca698d7 +size 5948053 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T115217414890Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T115217414890Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e16de2771249cff7d4576ab8d3900f4a5c43a06 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T115217414890Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103575343104, + "active_bytes_peak_delta": 23644738048, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575343104, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103223021568, + "device": 3, + "inactive_split_bytes_current": 3574268928, + "inactive_split_bytes_peak": 5561548288, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T11:52:17.414487Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223021568, + "allocated_bytes_peak_delta": 23644738048, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 79930605056, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 79578283520, + "device": 3, + "inactive_split_bytes_current": 2691367936, + "inactive_split_bytes_peak": 5561548288, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 96118767616, + "reserved_bytes_peak": 96118767616, + "segment_count_current": 509, + "segment_count_peak": 509, + "timestamp_utc": "2026-08-01T11:52:16.435499Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3574268928, + "inactive_split_bytes_current_delta": 882900992, + "inactive_split_bytes_peak": 5561548288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106380132352, + "reserved_bytes_current_delta": 10261364736, + "reserved_bytes_peak": 106380132352, + "reserved_bytes_peak_delta": 10261364736, + "segment_count_current": 578, + "segment_count_current_delta": 69, + "segment_count_peak": 578, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T115217414890Z.pickle", + "timestamp_utc": "2026-08-01T11:52:17.529273Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T115217414890Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T115217414890Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b080d6156d90419b7667ca0606c55118f27d70b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T115217414890Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02e2bde69f16f49e81a0b04a9559abda188672c71ca03bdfe1adb059fe0747c3 +size 5942050 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T115218477855Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T115218477855Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3884c74aef5244924dd4b52d4c171b7a95853ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T115218477855Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103575343104, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575343104, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103223021568, + "device": 3, + "inactive_split_bytes_current": 3574268928, + "inactive_split_bytes_peak": 5561548288, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T11:52:18.477458Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223021568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575343104, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103223021568, + "device": 3, + "inactive_split_bytes_current": 3574268928, + "inactive_split_bytes_peak": 5561548288, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T11:52:17.545295Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3574268928, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5561548288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106380132352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106380132352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 578, + "segment_count_current_delta": 0, + "segment_count_peak": 578, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T115218477855Z.pickle", + "timestamp_utc": "2026-08-01T11:52:18.591075Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T115218477855Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T115218477855Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b07a813f9d1964b4986f009f217f4fb8173ceedb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T115218477855Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71ba75f8776c6d97bb6ceddd076e69f428cf33bc02f52f29734d7dd9221dd6bb +size 5926407 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T115221499976Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T115221499976Z.json new file mode 100644 index 0000000000000000000000000000000000000000..603839c9f1ec995071b954c1403fe5eaba286e30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T115221499976Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 160826097664, + "active_bytes_peak_delta": 57250754560, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_peak": 10921235456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 175437250560, + "reserved_bytes_peak": 175437250560, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:21.499438Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103223729664, + "allocated_bytes_peak_delta": 708096, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575343104, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103223021568, + "device": 3, + "inactive_split_bytes_current": 3574268928, + "inactive_split_bytes_peak": 5561548288, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T11:52:18.606968Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_current_delta": 3800952832, + "inactive_split_bytes_peak": 10921235456, + "inactive_split_bytes_peak_delta": 5359687168, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 175437250560, + "reserved_bytes_current_delta": 69057118208, + "reserved_bytes_peak": 175437250560, + "reserved_bytes_peak_delta": 69057118208, + "segment_count_current": 905, + "segment_count_current_delta": 327, + "segment_count_peak": 905, + "segment_count_peak_delta": 327, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T115221499976Z.pickle", + "timestamp_utc": "2026-08-01T11:52:21.626100Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T115221499976Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T115221499976Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fff4e063901d2c8219cae6fc6823659bda797c51 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T115221499976Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6acf34f7060d900ec8c43a9dc8d53e935acd87572898c52bcde5af9ed3c7b325 +size 6954236 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T115222449262Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T115222449262Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2f103797505cac7f39103448ac7b4c62b6270d73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T115222449262Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160826097664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_peak": 10921235456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 175437250560, + "reserved_bytes_peak": 175437250560, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:22.448776Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223729664, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_peak": 10921235456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 175437250560, + "reserved_bytes_peak": 175437250560, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:21.643908Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10921235456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 175437250560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175437250560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T115222449262Z.pickle", + "timestamp_utc": "2026-08-01T11:52:22.456027Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T115222449262Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T115222449262Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3fb2d9a549a36dbbc17bfc82d22426baa806c63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T115222449262Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37bfff1d02a8c21c29aebd96b91119a974a95b6af7bdb4654e3d61437c4fac21 +size 247066 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T115223256662Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T115223256662Z.json new file mode 100644 index 0000000000000000000000000000000000000000..adc48880b1b0bf6de7329630522a3822cbcd0526 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T115223256662Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160826097664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_peak": 10921235456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 175437250560, + "reserved_bytes_peak": 175437250560, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:23.256247Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223729664, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_peak": 10921235456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 175437250560, + "reserved_bytes_peak": 175437250560, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:22.456982Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10921235456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 175437250560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175437250560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T115223256662Z.pickle", + "timestamp_utc": "2026-08-01T11:52:23.262968Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T115223256662Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T115223256662Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e4a155a85164db2c8d754174d7d15fac4137c4f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T115223256662Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b253ffd1fd783e2e20ff021650b0c48c6642f5f62014af9d4d96fe9500d98997 +size 247066 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T115224063245Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T115224063245Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef23b598267e67e2dbb6b02f1bc3aee6822d306e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T115224063245Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160826097664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_peak": 10921235456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 175437250560, + "reserved_bytes_peak": 175437250560, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:24.062847Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223729664, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_peak": 10921235456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 175437250560, + "reserved_bytes_peak": 175437250560, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:23.263900Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10921235456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 175437250560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175437250560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T115224063245Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.069047Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T115224063245Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T115224063245Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e92362987950b1ffcd66fe7d6f3b2c9e9f64df84 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T115224063245Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a27ed3683b4d31fbc2b9951a22a4193e2b7c4924c25705ef053dc46ebd4c6c6 +size 247066 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T115224870138Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T115224870138Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b010f2922b710b87905894e3bcb059c88cae84ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T115224870138Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160826097664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_peak": 10921235456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 175437250560, + "reserved_bytes_peak": 175437250560, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:24.869742Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223729664, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_peak": 10921235456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 175437250560, + "reserved_bytes_peak": 175437250560, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:24.069981Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10921235456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 175437250560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175437250560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T115224870138Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.876190Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T115224870138Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T115224870138Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..425a2ecb275f3e6d37ccaa514adf4924c65e713e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T115224870138Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8670912b8fb144848653060a60af4cc89d68695417af3d2c2ce5fda8a2e25ebc +size 247066 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T115225679002Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T115225679002Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34c1393d09ab8e10f6a91d8273699722797a93b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T115225679002Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160826097664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_peak": 10921235456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 175437250560, + "reserved_bytes_peak": 175437250560, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:25.678606Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103223729664, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_peak": 10921235456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 175437250560, + "reserved_bytes_peak": 175437250560, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:24.877125Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10921235456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 175437250560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175437250560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 905, + "segment_count_current_delta": 0, + "segment_count_peak": 905, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T115225679002Z.pickle", + "timestamp_utc": "2026-08-01T11:52:25.684741Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T115225679002Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T115225679002Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a95b26a3af8a7e607e8505ef18f995eb69709c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T115225679002Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1301d9735d02addcd0ed7fd2906d4cccdc20adf9af9a9854558d32a533a71e99 +size 247066 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T115314466636Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T115314466636Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9459786d0114b693e37fc0999d7b3397aee85e06 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T115314466636Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61218657280, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 160826097664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61218657280, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 61218657280, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 3505744896, + "inactive_split_bytes_peak": 10921235456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 176425009152, + "reserved_bytes_peak": 176425009152, + "segment_count_current": 914, + "segment_count_peak": 914, + "timestamp_utc": "2026-08-01T11:53:14.466157Z", + "world_size": 8 + }, + "allocated_bytes_current": 61218657280, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103223729664, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160826097664, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103223729664, + "device": 3, + "inactive_split_bytes_current": 7375221760, + "inactive_split_bytes_peak": 10921235456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 175437250560, + "reserved_bytes_peak": 175437250560, + "segment_count_current": 905, + "segment_count_peak": 905, + "timestamp_utc": "2026-08-01T11:52:25.685629Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3505744896, + "inactive_split_bytes_current_delta": -3869476864, + "inactive_split_bytes_peak": 10921235456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 176425009152, + "reserved_bytes_current_delta": 987758592, + "reserved_bytes_peak": 176425009152, + "reserved_bytes_peak_delta": 987758592, + "segment_count_current": 914, + "segment_count_current_delta": 9, + "segment_count_peak": 914, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T115314466636Z.pickle", + "timestamp_utc": "2026-08-01T11:53:14.553022Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T115314466636Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T115314466636Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..24ea170ed458f1f244d3c6574bf7754e69df0178 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T115314466636Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0607edc496d3c174ece0f46cf72bc390d0e620e0069828b5e9778617b237fc11 +size 4058004 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T115216293028Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T115216293028Z.json new file mode 100644 index 0000000000000000000000000000000000000000..049c4ddb9f9beb9103b39069784d1ad2218adac5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T115216293028Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459454464, + "active_bytes_current_delta": 37227556864, + "active_bytes_peak": 79938184704, + "active_bytes_peak_delta": 44471406080, + "after": { + "active_bytes_current": 72459454464, + "active_bytes_peak": 79938184704, + "allocated_bytes_current": 72459454464, + "allocated_bytes_peak": 79585863168, + "device": 4, + "inactive_split_bytes_current": 2706667520, + "inactive_split_bytes_peak": 5443141120, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 95883886592, + "reserved_bytes_peak": 95883886592, + "segment_count_current": 508, + "segment_count_peak": 508, + "timestamp_utc": "2026-08-01T11:52:16.292435Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459454464, + "allocated_bytes_current_delta": 37227556864, + "allocated_bytes_peak": 79585863168, + "allocated_bytes_peak_delta": 44119084544, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:51:07.374385Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2706667520, + "inactive_split_bytes_current_delta": 2574290944, + "inactive_split_bytes_peak": 5443141120, + "inactive_split_bytes_peak_delta": 5289731584, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 95883886592, + "reserved_bytes_current_delta": 56639881216, + "reserved_bytes_peak": 95883886592, + "reserved_bytes_peak_delta": 56639881216, + "segment_count_current": 508, + "segment_count_current_delta": 437, + "segment_count_peak": 508, + "segment_count_peak_delta": 437, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T115216293028Z.pickle", + "timestamp_utc": "2026-08-01T11:52:16.413079Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T115216293028Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T115216293028Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b2e2afb3e6efce5b81d63777d8efc3d313599b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T115216293028Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32352f4ec2f7a1830c99f9a7fd67ac7d0072812b78e809127690c260389beb33 +size 5956360 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T115217414855Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T115217414855Z.json new file mode 100644 index 0000000000000000000000000000000000000000..24978f500b64e699d82c5fa42d3bc99349cdd75e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T115217414855Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459454464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103569223680, + "active_bytes_peak_delta": 23631038976, + "after": { + "active_bytes_current": 72459454464, + "active_bytes_peak": 103569223680, + "allocated_bytes_current": 72459454464, + "allocated_bytes_peak": 103216902144, + "device": 4, + "inactive_split_bytes_current": 3235149824, + "inactive_split_bytes_peak": 5443141120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106392715264, + "reserved_bytes_peak": 106392715264, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T11:52:17.414444Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459454464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103216902144, + "allocated_bytes_peak_delta": 23631038976, + "before": { + "active_bytes_current": 72459454464, + "active_bytes_peak": 79938184704, + "allocated_bytes_current": 72459454464, + "allocated_bytes_peak": 79585863168, + "device": 4, + "inactive_split_bytes_current": 2706667520, + "inactive_split_bytes_peak": 5443141120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 95883886592, + "reserved_bytes_peak": 95883886592, + "segment_count_current": 508, + "segment_count_peak": 508, + "timestamp_utc": "2026-08-01T11:52:16.432386Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3235149824, + "inactive_split_bytes_current_delta": 528482304, + "inactive_split_bytes_peak": 5443141120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106392715264, + "reserved_bytes_current_delta": 10508828672, + "reserved_bytes_peak": 106392715264, + "reserved_bytes_peak_delta": 10508828672, + "segment_count_current": 580, + "segment_count_current_delta": 72, + "segment_count_peak": 580, + "segment_count_peak_delta": 72, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T115217414855Z.pickle", + "timestamp_utc": "2026-08-01T11:52:17.529058Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T115217414855Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T115217414855Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2adb29037c0dff8faa15e17386fc1f4e9641328f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T115217414855Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a12d12f0ef7bc6b05810fcbcad166528e00d1bd3b7462a9bc17f99479203b896 +size 6616767 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T115218477830Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T115218477830Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9f60bc8173afbc2039ad2501191609b7041a34bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T115218477830Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459454464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103569224704, + "active_bytes_peak_delta": 1024, + "after": { + "active_bytes_current": 72459454464, + "active_bytes_peak": 103569224704, + "allocated_bytes_current": 72459454464, + "allocated_bytes_peak": 103216903168, + "device": 4, + "inactive_split_bytes_current": 3532945408, + "inactive_split_bytes_peak": 5443141120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106392715264, + "reserved_bytes_peak": 106392715264, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T11:52:18.477423Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459454464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103216903168, + "allocated_bytes_peak_delta": 1024, + "before": { + "active_bytes_current": 72459454464, + "active_bytes_peak": 103569223680, + "allocated_bytes_current": 72459454464, + "allocated_bytes_peak": 103216902144, + "device": 4, + "inactive_split_bytes_current": 3235149824, + "inactive_split_bytes_peak": 5443141120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106392715264, + "reserved_bytes_peak": 106392715264, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T11:52:17.545293Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3532945408, + "inactive_split_bytes_current_delta": 297795584, + "inactive_split_bytes_peak": 5443141120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106392715264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106392715264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T115218477830Z.pickle", + "timestamp_utc": "2026-08-01T11:52:18.593427Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T115218477830Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T115218477830Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a0bad55dc71350acc147d6abc3a4087eb898af66 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T115218477830Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e4085cc3230cfb197a5a141affa14374b9159b248829a7d374423c85f072efb +size 6629880 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T115221499961Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T115221499961Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e425193d7d465f91d99b196f7b156ffc700dfbd1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T115221499961Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72477415424, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 160839240704, + "active_bytes_peak_delta": 57270016000, + "after": { + "active_bytes_current": 72477415424, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 72477415424, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_peak": 10940729344, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 175397404672, + "reserved_bytes_peak": 175397404672, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-01T11:52:21.499423Z", + "world_size": 8 + }, + "allocated_bytes_current": 72477415424, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103225731584, + "allocated_bytes_peak_delta": 8828416, + "before": { + "active_bytes_current": 72459454464, + "active_bytes_peak": 103569224704, + "allocated_bytes_current": 72459454464, + "allocated_bytes_peak": 103216903168, + "device": 4, + "inactive_split_bytes_current": 3532945408, + "inactive_split_bytes_peak": 5443141120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106392715264, + "reserved_bytes_peak": 106392715264, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T11:52:18.609038Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_current_delta": 3744329728, + "inactive_split_bytes_peak": 10940729344, + "inactive_split_bytes_peak_delta": 5497588224, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 175397404672, + "reserved_bytes_current_delta": 69004689408, + "reserved_bytes_peak": 175397404672, + "reserved_bytes_peak_delta": 69004689408, + "segment_count_current": 911, + "segment_count_current_delta": 331, + "segment_count_peak": 911, + "segment_count_peak_delta": 331, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T115221499961Z.pickle", + "timestamp_utc": "2026-08-01T11:52:21.625534Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T115221499961Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T115221499961Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..74763ddf42c4681e64e78a67f1b9667e9df42851 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T115221499961Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:080fc16574c6fe05468b1a7b9dd1f55d563219bc14663fcab13fb0d19437c497 +size 6958498 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T115222449230Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T115222449230Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f381ebe4dfe4fd426f26b0c2c14c22ca934f50a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T115222449230Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72477415424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160839240704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72477415424, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 72477415424, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_peak": 10940729344, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 175397404672, + "reserved_bytes_peak": 175397404672, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-01T11:52:22.448734Z", + "world_size": 8 + }, + "allocated_bytes_current": 72477415424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103225731584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72477415424, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 72477415424, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_peak": 10940729344, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 175397404672, + "reserved_bytes_peak": 175397404672, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-01T11:52:21.645520Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10940729344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 175397404672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175397404672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 911, + "segment_count_current_delta": 0, + "segment_count_peak": 911, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T115222449230Z.pickle", + "timestamp_utc": "2026-08-01T11:52:22.456254Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T115222449230Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T115222449230Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e50172af7248adeb0d12d1f163b0438de8323de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T115222449230Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e089ac0e8610a93269640f8be3a51dbf507e383a99826aa318e60c1d067115f9 +size 250358 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T115223256631Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T115223256631Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef788a2382ab2b7fd8f61089514218cd70b2cf11 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T115223256631Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72477415424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160839240704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72477415424, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 72477415424, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_peak": 10940729344, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 175397404672, + "reserved_bytes_peak": 175397404672, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-01T11:52:23.256218Z", + "world_size": 8 + }, + "allocated_bytes_current": 72477415424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103225731584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72477415424, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 72477415424, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_peak": 10940729344, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 175397404672, + "reserved_bytes_peak": 175397404672, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-01T11:52:22.457240Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10940729344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 175397404672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175397404672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 911, + "segment_count_current_delta": 0, + "segment_count_peak": 911, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T115223256631Z.pickle", + "timestamp_utc": "2026-08-01T11:52:23.262857Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T115223256631Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T115223256631Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..26e8cfd65cdc3997a982878a6dd007c1c096dd7a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T115223256631Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c0e567f1a56683d243fb0da95b9213bdbdd4b54efaf067338a13cfd64e10131 +size 250358 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T115224063236Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T115224063236Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8291cf29ebeae5b88abf754a5c3437316f9bc9ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T115224063236Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72477415424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160839240704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72477415424, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 72477415424, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_peak": 10940729344, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 175397404672, + "reserved_bytes_peak": 175397404672, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-01T11:52:24.062837Z", + "world_size": 8 + }, + "allocated_bytes_current": 72477415424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103225731584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72477415424, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 72477415424, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_peak": 10940729344, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 175397404672, + "reserved_bytes_peak": 175397404672, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-01T11:52:23.263784Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10940729344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 175397404672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175397404672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 911, + "segment_count_current_delta": 0, + "segment_count_peak": 911, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T115224063236Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.069296Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T115224063236Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T115224063236Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..011e65c7d0605a27325f71436bd7d7b75cb10488 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T115224063236Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ecb73801fe6a215bc78534b890aad68d45aa6ac1ecc873d38145f96bc0545f3 +size 250358 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T115224870342Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T115224870342Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6a7b972f6c0edd3eb431477bc6791e5991c935c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T115224870342Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72477415424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160839240704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72477415424, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 72477415424, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_peak": 10940729344, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 175397404672, + "reserved_bytes_peak": 175397404672, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-01T11:52:24.869752Z", + "world_size": 8 + }, + "allocated_bytes_current": 72477415424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103225731584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72477415424, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 72477415424, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_peak": 10940729344, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 175397404672, + "reserved_bytes_peak": 175397404672, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-01T11:52:24.070252Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10940729344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 175397404672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175397404672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 911, + "segment_count_current_delta": 0, + "segment_count_peak": 911, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T115224870342Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.876832Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T115224870342Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T115224870342Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4261ff1e92f8e7efa5efa69a17acc39c93626d07 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T115224870342Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:638a89166e37b52432bac20d2c10f661bd355213b790a252e5bf858206f11c66 +size 250358 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T115225678922Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T115225678922Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ab63a77667f1cdaf9afec90bbf6c30d63191583 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T115225678922Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72477415424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160839240704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72477415424, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 72477415424, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_peak": 10940729344, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 175397404672, + "reserved_bytes_peak": 175397404672, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-01T11:52:25.678530Z", + "world_size": 8 + }, + "allocated_bytes_current": 72477415424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103225731584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72477415424, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 72477415424, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_peak": 10940729344, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 175397404672, + "reserved_bytes_peak": 175397404672, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-01T11:52:24.877825Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10940729344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 175397404672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175397404672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 911, + "segment_count_current_delta": 0, + "segment_count_peak": 911, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T115225678922Z.pickle", + "timestamp_utc": "2026-08-01T11:52:25.684774Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T115225678922Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T115225678922Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f8b85a6b44c26ad57024aa57ee8aed7b919202f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T115225678922Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2224bb347ef80d56ac912424e4e4e7501e7f3e45ee62589ca1d9f9136f1a326e +size 250358 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T115314466642Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T115314466642Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c3030831e7def372bb7c001376f80d5194df2a34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T115314466642Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61220134912, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 160839240704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61220134912, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 61220134912, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 3200180224, + "inactive_split_bytes_peak": 10940729344, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 176854925312, + "reserved_bytes_peak": 176854925312, + "segment_count_current": 922, + "segment_count_peak": 922, + "timestamp_utc": "2026-08-01T11:53:14.466158Z", + "world_size": 8 + }, + "allocated_bytes_current": 61220134912, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103225731584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72477415424, + "active_bytes_peak": 160839240704, + "allocated_bytes_current": 72477415424, + "allocated_bytes_peak": 103225731584, + "device": 4, + "inactive_split_bytes_current": 7277275136, + "inactive_split_bytes_peak": 10940729344, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 175397404672, + "reserved_bytes_peak": 175397404672, + "segment_count_current": 911, + "segment_count_peak": 911, + "timestamp_utc": "2026-08-01T11:52:25.685710Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3200180224, + "inactive_split_bytes_current_delta": -4077094912, + "inactive_split_bytes_peak": 10940729344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 176854925312, + "reserved_bytes_current_delta": 1457520640, + "reserved_bytes_peak": 176854925312, + "reserved_bytes_peak_delta": 1457520640, + "segment_count_current": 922, + "segment_count_current_delta": 11, + "segment_count_peak": 922, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T115314466642Z.pickle", + "timestamp_utc": "2026-08-01T11:53:14.549695Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T115314466642Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T115314466642Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d4b85d797981978f93a697cc36299674df0466a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T115314466642Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b034d7b3a58495ca0baa772d10f8ebdced7eb734b261429e0d804c331925b90d +size 4200022 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T115216293074Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T115216293074Z.json new file mode 100644 index 0000000000000000000000000000000000000000..498a1e5ba7170acfb4ec2371ed3f6d9be449f47c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T115216293074Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 37226521600, + "active_bytes_peak": 80287061504, + "active_bytes_peak_delta": 44820725248, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 80287061504, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 79582477824, + "device": 5, + "inactive_split_bytes_current": 2808808448, + "inactive_split_bytes_peak": 5561548288, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 96353648640, + "reserved_bytes_peak": 96353648640, + "segment_count_current": 507, + "segment_count_peak": 507, + "timestamp_utc": "2026-08-01T11:52:16.292451Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 37226521600, + "allocated_bytes_peak": 79582477824, + "allocated_bytes_peak_delta": 44116141568, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:51:07.374622Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2808808448, + "inactive_split_bytes_current_delta": 2675989504, + "inactive_split_bytes_peak": 5561548288, + "inactive_split_bytes_peak_delta": 5401162240, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 96353648640, + "reserved_bytes_current_delta": 57109643264, + "reserved_bytes_peak": 96353648640, + "reserved_bytes_peak_delta": 57109643264, + "segment_count_current": 507, + "segment_count_current_delta": 436, + "segment_count_peak": 507, + "segment_count_peak_delta": 436, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T115216293074Z.pickle", + "timestamp_utc": "2026-08-01T11:52:16.413476Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T115216293074Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T115216293074Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..632c303ca5a5f391ec943a916b8658107d706f50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T115216293074Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a5ad405d0ad20cf9bdf2b6cbf5620f84c39536d0e0ea1e47ae63ef7256d46a0 +size 5947792 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T115217414889Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T115217414889Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ad0de76c4d54c7a6391b94612e29023b69e4de9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T115217414889Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103575080960, + "active_bytes_peak_delta": 23288019456, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575080960, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103222759424, + "device": 5, + "inactive_split_bytes_current": 3809149952, + "inactive_split_bytes_peak": 5561548288, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T11:52:17.414475Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103222759424, + "allocated_bytes_peak_delta": 23640281600, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 80287061504, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 79582477824, + "device": 5, + "inactive_split_bytes_current": 2808808448, + "inactive_split_bytes_peak": 5561548288, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 96353648640, + "reserved_bytes_peak": 96353648640, + "segment_count_current": 507, + "segment_count_peak": 507, + "timestamp_utc": "2026-08-01T11:52:16.435133Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3809149952, + "inactive_split_bytes_current_delta": 1000341504, + "inactive_split_bytes_peak": 5561548288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106380132352, + "reserved_bytes_current_delta": 10026483712, + "reserved_bytes_peak": 106380132352, + "reserved_bytes_peak_delta": 10026483712, + "segment_count_current": 578, + "segment_count_current_delta": 71, + "segment_count_peak": 578, + "segment_count_peak_delta": 71, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T115217414889Z.pickle", + "timestamp_utc": "2026-08-01T11:52:17.523802Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T115217414889Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T115217414889Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..884cab11a7bf195dadfcd528949fd1a0a1e4cf8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T115217414889Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23cfb8bb5046c175a034fd312dbf3475ef7b92ea1c1fc07a1a066361e97f58d4 +size 5928632 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T115218477811Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T115218477811Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9784a5695ba71ce217275ae1bffbf11232ff0494 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T115218477811Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103575080960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575080960, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103222759424, + "device": 5, + "inactive_split_bytes_current": 3666543616, + "inactive_split_bytes_peak": 5561548288, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T11:52:18.477423Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103222759424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575080960, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103222759424, + "device": 5, + "inactive_split_bytes_current": 3809149952, + "inactive_split_bytes_peak": 5561548288, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T11:52:17.539846Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3666543616, + "inactive_split_bytes_current_delta": -142606336, + "inactive_split_bytes_peak": 5561548288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106380132352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106380132352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 578, + "segment_count_current_delta": 0, + "segment_count_peak": 578, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T115218477811Z.pickle", + "timestamp_utc": "2026-08-01T11:52:18.590018Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T115218477811Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T115218477811Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76eaf16280d2c1b76deb4fb0759ede4f3e4db8d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T115218477811Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47d4d3a8e0db94d660a1fe006ab6015189761f9504d992fd149d5288dfeedbc4 +size 5938953 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T115221499942Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T115221499942Z.json new file mode 100644 index 0000000000000000000000000000000000000000..55df5651976e6be464c7f586bc15590f752c1056 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T115221499942Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 160846282752, + "active_bytes_peak_delta": 57271201792, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_peak": 11747513344, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 175646965760, + "reserved_bytes_peak": 175646965760, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:21.499413Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103232512000, + "allocated_bytes_peak_delta": 9752576, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103575080960, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103222759424, + "device": 5, + "inactive_split_bytes_current": 3666543616, + "inactive_split_bytes_peak": 5561548288, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T11:52:18.605869Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_current_delta": 4239257600, + "inactive_split_bytes_peak": 11747513344, + "inactive_split_bytes_peak_delta": 6185965056, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 175646965760, + "reserved_bytes_current_delta": 69266833408, + "reserved_bytes_peak": 175646965760, + "reserved_bytes_peak_delta": 69266833408, + "segment_count_current": 907, + "segment_count_current_delta": 329, + "segment_count_peak": 907, + "segment_count_peak_delta": 329, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T115221499942Z.pickle", + "timestamp_utc": "2026-08-01T11:52:21.625085Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T115221499942Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T115221499942Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..39484c79dfa9279c2219c655be8eb8864d0c5a0b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T115221499942Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7444f521592c38e8e8b700d49ed737a2b15e1d538d24df1d6d1e3255589e85e5 +size 6955336 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T115222449236Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T115222449236Z.json new file mode 100644 index 0000000000000000000000000000000000000000..912701aed748a88d92866ed42db3141b143b2e7b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T115222449236Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160846282752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_peak": 11747513344, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 175646965760, + "reserved_bytes_peak": 175646965760, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:22.448758Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232512000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_peak": 11747513344, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 175646965760, + "reserved_bytes_peak": 175646965760, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:21.645055Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11747513344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 175646965760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175646965760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 907, + "segment_count_current_delta": 0, + "segment_count_peak": 907, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T115222449236Z.pickle", + "timestamp_utc": "2026-08-01T11:52:22.456299Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T115222449236Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T115222449236Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..afdcc5f79ac23861b63d144a1815692bc194353f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T115222449236Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:116cd185d8c3f81df0780f63cc8851b6d08aa0cdba9ad94446901a1384402180 +size 247743 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T115223256655Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T115223256655Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f9dad601941be5d7fd137d24ed8b1cf39e565b2e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T115223256655Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160846282752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_peak": 11747513344, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 175646965760, + "reserved_bytes_peak": 175646965760, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:23.256242Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232512000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_peak": 11747513344, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 175646965760, + "reserved_bytes_peak": 175646965760, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:22.457284Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11747513344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 175646965760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175646965760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 907, + "segment_count_current_delta": 0, + "segment_count_peak": 907, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T115223256655Z.pickle", + "timestamp_utc": "2026-08-01T11:52:23.262924Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T115223256655Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T115223256655Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..176606fba91c7fc451938b8df428715d6a7a7721 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T115223256655Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cf306da69f80e1abe7bcbddc1aba543a210f4bd50edbc65e37f1a58b21df6e3 +size 247743 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T115224063201Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T115224063201Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a35c06ed5aa20027e60f3dfa3abff14840871e83 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T115224063201Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160846282752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_peak": 11747513344, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 175646965760, + "reserved_bytes_peak": 175646965760, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:24.062804Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232512000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_peak": 11747513344, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 175646965760, + "reserved_bytes_peak": 175646965760, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:23.263845Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11747513344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 175646965760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175646965760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 907, + "segment_count_current_delta": 0, + "segment_count_peak": 907, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T115224063201Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.069189Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T115224063201Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T115224063201Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8cefb6e3ee60051ec65c3316d50c1c29b04a3ff1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T115224063201Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e0772d43e7f42cd3f4d6335e487ec166365125b11dfbf7e383a7fddede14571 +size 247743 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T115224870135Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T115224870135Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fa606df8413790dcc5a37e57c0a20e7bebea3aa1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T115224870135Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160846282752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_peak": 11747513344, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 175646965760, + "reserved_bytes_peak": 175646965760, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:24.869725Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232512000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_peak": 11747513344, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 175646965760, + "reserved_bytes_peak": 175646965760, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:24.070113Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11747513344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 175646965760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175646965760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 907, + "segment_count_current_delta": 0, + "segment_count_peak": 907, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T115224870135Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.876239Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T115224870135Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T115224870135Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e9e81762cb56474be2e17564f0e391ebf0f1a40 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T115224870135Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4848e6c8b84fdf934349124d7529c541189992d48e40b353e381545927f4459f +size 247743 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T115225678952Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T115225678952Z.json new file mode 100644 index 0000000000000000000000000000000000000000..538a66ca4e6441ad8bbc41acea49cd995e765c2d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T115225678952Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160846282752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_peak": 11747513344, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 175646965760, + "reserved_bytes_peak": 175646965760, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:25.678558Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103232512000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_peak": 11747513344, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 175646965760, + "reserved_bytes_peak": 175646965760, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:24.877194Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11747513344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 175646965760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175646965760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 907, + "segment_count_current_delta": 0, + "segment_count_peak": 907, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T115225678952Z.pickle", + "timestamp_utc": "2026-08-01T11:52:25.684833Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T115225678952Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T115225678952Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d84ccbea7e64ea7269112476363f23cc9db7e10 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T115225678952Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:085d3d11b3b542da01bd3e0120214becb9aa2f92a540b4962f71a83901311b39 +size 247743 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T115314466643Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T115314466643Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f432213c61cd263ccc325748941843824f1ec74 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T115314466643Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61218657280, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 160846282752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61218657280, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 61218657280, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 3918883840, + "inactive_split_bytes_peak": 11747513344, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 176869605376, + "reserved_bytes_peak": 176869605376, + "segment_count_current": 917, + "segment_count_peak": 917, + "timestamp_utc": "2026-08-01T11:53:14.466166Z", + "world_size": 8 + }, + "allocated_bytes_current": 61218657280, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103232512000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160846282752, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103232512000, + "device": 5, + "inactive_split_bytes_current": 7905801216, + "inactive_split_bytes_peak": 11747513344, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 175646965760, + "reserved_bytes_peak": 175646965760, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:25.685735Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3918883840, + "inactive_split_bytes_current_delta": -3986917376, + "inactive_split_bytes_peak": 11747513344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 176869605376, + "reserved_bytes_current_delta": 1222639616, + "reserved_bytes_peak": 176869605376, + "reserved_bytes_peak_delta": 1222639616, + "segment_count_current": 917, + "segment_count_current_delta": 10, + "segment_count_peak": 917, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T115314466643Z.pickle", + "timestamp_utc": "2026-08-01T11:53:14.549645Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T115314466643Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T115314466643Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13539fc2b11264dfe146e6599a09886c9264a81a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T115314466643Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67d0c8100ebc811fbea7e7fd2de77a360044e07b67082a67ff956197396d9812 +size 4058709 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T115216293743Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T115216293743Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d059d5f6635cdf791278c8f20e50de6ec4e1c382 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T115216293743Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 37226521600, + "active_bytes_peak": 79930545664, + "active_bytes_peak_delta": 44464209408, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 79930545664, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 79578283520, + "device": 6, + "inactive_split_bytes_current": 2808808448, + "inactive_split_bytes_peak": 5444107776, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 96823410688, + "reserved_bytes_peak": 96823410688, + "segment_count_current": 514, + "segment_count_peak": 514, + "timestamp_utc": "2026-08-01T11:52:16.292884Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 37226521600, + "allocated_bytes_peak": 79578283520, + "allocated_bytes_peak_delta": 44111947264, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:51:07.374955Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2808808448, + "inactive_split_bytes_current_delta": 2675989504, + "inactive_split_bytes_peak": 5444107776, + "inactive_split_bytes_peak_delta": 5283721728, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 96823410688, + "reserved_bytes_current_delta": 57579405312, + "reserved_bytes_peak": 96823410688, + "reserved_bytes_peak_delta": 57579405312, + "segment_count_current": 514, + "segment_count_current_delta": 443, + "segment_count_peak": 514, + "segment_count_peak_delta": 443, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T115216293743Z.pickle", + "timestamp_utc": "2026-08-01T11:52:16.418812Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T115216293743Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T115216293743Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..07a2ac577385c2aa75e91bd3cecaccc31e998be3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T115216293743Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa958651327b8c58fc227ff6881c3e3e592be68b97263e8560449fdeb582f7e1 +size 5948956 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T115217414941Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T115217414941Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8476df436386c743fd3e175178c465dcd74a5a68 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T115217414941Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103572661248, + "active_bytes_peak_delta": 23642115584, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103572661248, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103221215232, + "device": 6, + "inactive_split_bytes_current": 3224044544, + "inactive_split_bytes_peak": 5444107776, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106382229504, + "reserved_bytes_peak": 106382229504, + "segment_count_current": 585, + "segment_count_peak": 585, + "timestamp_utc": "2026-08-01T11:52:17.414496Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103221215232, + "allocated_bytes_peak_delta": 23642931712, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 79930545664, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 79578283520, + "device": 6, + "inactive_split_bytes_current": 2808808448, + "inactive_split_bytes_peak": 5444107776, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 96823410688, + "reserved_bytes_peak": 96823410688, + "segment_count_current": 514, + "segment_count_peak": 514, + "timestamp_utc": "2026-08-01T11:52:16.447335Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3224044544, + "inactive_split_bytes_current_delta": 415236096, + "inactive_split_bytes_peak": 5444107776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106382229504, + "reserved_bytes_current_delta": 9558818816, + "reserved_bytes_peak": 106382229504, + "reserved_bytes_peak_delta": 9558818816, + "segment_count_current": 585, + "segment_count_current_delta": 71, + "segment_count_peak": 585, + "segment_count_peak_delta": 71, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T115217414941Z.pickle", + "timestamp_utc": "2026-08-01T11:52:17.529303Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T115217414941Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T115217414941Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b909d2a4b40584f60b490e75cb8de6f36057fcaf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T115217414941Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa2534a4b4d942ad52cac1063b881d09c74839343434f4f40af1243522502333 +size 6618825 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T115218477826Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T115218477826Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dc4d563997622a586ea233cd15b3a6c0e0454aa5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T115218477826Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457976832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103572661248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103572661248, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103221215232, + "device": 6, + "inactive_split_bytes_current": 3402302464, + "inactive_split_bytes_peak": 5444107776, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106382229504, + "reserved_bytes_peak": 106382229504, + "segment_count_current": 585, + "segment_count_peak": 585, + "timestamp_utc": "2026-08-01T11:52:18.477430Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457976832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103221215232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103572661248, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103221215232, + "device": 6, + "inactive_split_bytes_current": 3224044544, + "inactive_split_bytes_peak": 5444107776, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106382229504, + "reserved_bytes_peak": 106382229504, + "segment_count_current": 585, + "segment_count_peak": 585, + "timestamp_utc": "2026-08-01T11:52:17.544343Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3402302464, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 5444107776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106382229504, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106382229504, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 585, + "segment_count_current_delta": 0, + "segment_count_peak": 585, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T115218477826Z.pickle", + "timestamp_utc": "2026-08-01T11:52:18.588675Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T115218477826Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T115218477826Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8bfbde2699f0d87deb4595e1ef9dbd00cf66e4c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T115218477826Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f03dff29dd94005ebe74bd097d45df83f40be797f2109ae7e127ef26a4ca6fe6 +size 6622700 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T115221499952Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T115221499952Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3237ee7dab48f769312467ae9e21280813aae877 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T115221499952Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 160840908800, + "active_bytes_peak_delta": 57268247552, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_peak": 11044967424, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 175206563840, + "reserved_bytes_peak": 175206563840, + "segment_count_current": 908, + "segment_count_peak": 908, + "timestamp_utc": "2026-08-01T11:52:21.499416Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103227138048, + "allocated_bytes_peak_delta": 5922816, + "before": { + "active_bytes_current": 72457976832, + "active_bytes_peak": 103572661248, + "allocated_bytes_current": 72457976832, + "allocated_bytes_peak": 103221215232, + "device": 6, + "inactive_split_bytes_current": 3402302464, + "inactive_split_bytes_peak": 5444107776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106382229504, + "reserved_bytes_peak": 106382229504, + "segment_count_current": 585, + "segment_count_peak": 585, + "timestamp_utc": "2026-08-01T11:52:18.602565Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_current_delta": 3861770240, + "inactive_split_bytes_peak": 11044967424, + "inactive_split_bytes_peak_delta": 5600859648, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 175206563840, + "reserved_bytes_current_delta": 68824334336, + "reserved_bytes_peak": 175206563840, + "reserved_bytes_peak_delta": 68824334336, + "segment_count_current": 908, + "segment_count_current_delta": 323, + "segment_count_peak": 908, + "segment_count_peak_delta": 323, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T115221499952Z.pickle", + "timestamp_utc": "2026-08-01T11:52:21.625324Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T115221499952Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T115221499952Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b4e4c3a062d7ca368ba367aa2912728fc41cb075 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T115221499952Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4baa64932319f716c24580cf9c1cb0ecf41046231f1dc7b6c219805557425776 +size 6956702 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T115222449255Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T115222449255Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e86ed6fedfa481bf3b58d45825179710453965d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T115222449255Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160840908800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_peak": 11044967424, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 175206563840, + "reserved_bytes_peak": 175206563840, + "segment_count_current": 908, + "segment_count_peak": 908, + "timestamp_utc": "2026-08-01T11:52:22.448775Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103227138048, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_peak": 11044967424, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 175206563840, + "reserved_bytes_peak": 175206563840, + "segment_count_current": 908, + "segment_count_peak": 908, + "timestamp_utc": "2026-08-01T11:52:21.640694Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11044967424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 175206563840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175206563840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 908, + "segment_count_current_delta": 0, + "segment_count_peak": 908, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T115222449255Z.pickle", + "timestamp_utc": "2026-08-01T11:52:22.455923Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T115222449255Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T115222449255Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a39dd6afe475d8387ce4fe7513e8fbead1f3641d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T115222449255Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65cbb902d7659a39b6a89a9ed7d3114e97fc2eeeff2791d2d61881f4e4706cea +size 249332 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T115223256967Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T115223256967Z.json new file mode 100644 index 0000000000000000000000000000000000000000..63100a9562485adfddd2ac896bb1083bda1d54f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T115223256967Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160840908800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_peak": 11044967424, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 175206563840, + "reserved_bytes_peak": 175206563840, + "segment_count_current": 908, + "segment_count_peak": 908, + "timestamp_utc": "2026-08-01T11:52:23.256504Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103227138048, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_peak": 11044967424, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 175206563840, + "reserved_bytes_peak": 175206563840, + "segment_count_current": 908, + "segment_count_peak": 908, + "timestamp_utc": "2026-08-01T11:52:22.456893Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11044967424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 175206563840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175206563840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 908, + "segment_count_current_delta": 0, + "segment_count_peak": 908, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T115223256967Z.pickle", + "timestamp_utc": "2026-08-01T11:52:23.266666Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T115223256967Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T115223256967Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a54f6dc109b1e61ff670f8d0b51097d7ca78985d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T115223256967Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d94a97e802788858596ae17f10e785f1084d3b5e189773348924e090d2eae10 +size 249363 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T115224063279Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T115224063279Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a175de46f6ff79f915aede4b2fc2847a04b6fc29 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T115224063279Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160840908800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_peak": 11044967424, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 175206563840, + "reserved_bytes_peak": 175206563840, + "segment_count_current": 908, + "segment_count_peak": 908, + "timestamp_utc": "2026-08-01T11:52:24.062872Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103227138048, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_peak": 11044967424, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 175206563840, + "reserved_bytes_peak": 175206563840, + "segment_count_current": 908, + "segment_count_peak": 908, + "timestamp_utc": "2026-08-01T11:52:23.267662Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11044967424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 175206563840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175206563840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 908, + "segment_count_current_delta": 0, + "segment_count_peak": 908, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T115224063279Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.069341Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T115224063279Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T115224063279Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..54046d674f998c2be11b68e609855de97bd67d31 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T115224063279Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3039679ed9e4390a0451d0bc66b175ee27fb10c7e5b16098a859980e4760e77 +size 249332 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T115224870131Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T115224870131Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d89d2d03231f627418cb29e10f7785562eb2852 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T115224870131Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160840908800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_peak": 11044967424, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 175206563840, + "reserved_bytes_peak": 175206563840, + "segment_count_current": 908, + "segment_count_peak": 908, + "timestamp_utc": "2026-08-01T11:52:24.869725Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103227138048, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_peak": 11044967424, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 175206563840, + "reserved_bytes_peak": 175206563840, + "segment_count_current": 908, + "segment_count_peak": 908, + "timestamp_utc": "2026-08-01T11:52:24.070309Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11044967424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 175206563840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175206563840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 908, + "segment_count_current_delta": 0, + "segment_count_peak": 908, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T115224870131Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.876268Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T115224870131Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T115224870131Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e9db8f745db8c3763e8f51b0bb84304725dff84a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T115224870131Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23d9743558862f1103e2befd6c6ea2af74d40643ffae15d16d4b87bc40ad04f7 +size 249363 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T115225679369Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T115225679369Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7e86de79a9f896de7ca8fde279685cb4cafe287 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T115225679369Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475937792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160840908800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_peak": 11044967424, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 175206563840, + "reserved_bytes_peak": 175206563840, + "segment_count_current": 908, + "segment_count_peak": 908, + "timestamp_utc": "2026-08-01T11:52:25.678839Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475937792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103227138048, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_peak": 11044967424, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 175206563840, + "reserved_bytes_peak": 175206563840, + "segment_count_current": 908, + "segment_count_peak": 908, + "timestamp_utc": "2026-08-01T11:52:24.877238Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11044967424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 175206563840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175206563840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 908, + "segment_count_current_delta": 0, + "segment_count_peak": 908, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T115225679369Z.pickle", + "timestamp_utc": "2026-08-01T11:52:25.688960Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T115225679369Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T115225679369Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..57fdf392e0d0323633f7bf32bf78e9cfb1c9df1c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T115225679369Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4f0d494fa28c485b3d3de0a86180518d8d8acf48da6ae4368fee4c35a21c351 +size 249332 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T115314466631Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T115314466631Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc010f430d697c09d9e77b2f403ee606dcf143c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T115314466631Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61218657280, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 160840908800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61218657280, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 61218657280, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 3304418304, + "inactive_split_bytes_peak": 11044967424, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 176664084480, + "reserved_bytes_peak": 176664084480, + "segment_count_current": 919, + "segment_count_peak": 919, + "timestamp_utc": "2026-08-01T11:53:14.466150Z", + "world_size": 8 + }, + "allocated_bytes_current": 61218657280, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103227138048, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475937792, + "active_bytes_peak": 160840908800, + "allocated_bytes_current": 72475937792, + "allocated_bytes_peak": 103227138048, + "device": 6, + "inactive_split_bytes_current": 7264072704, + "inactive_split_bytes_peak": 11044967424, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 175206563840, + "reserved_bytes_peak": 175206563840, + "segment_count_current": 908, + "segment_count_peak": 908, + "timestamp_utc": "2026-08-01T11:52:25.690013Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3304418304, + "inactive_split_bytes_current_delta": -3959654400, + "inactive_split_bytes_peak": 11044967424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 176664084480, + "reserved_bytes_current_delta": 1457520640, + "reserved_bytes_peak": 176664084480, + "reserved_bytes_peak_delta": 1457520640, + "segment_count_current": 919, + "segment_count_current_delta": 11, + "segment_count_peak": 919, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T115314466631Z.pickle", + "timestamp_utc": "2026-08-01T11:53:14.552974Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T115314466631Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T115314466631Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..294df31a08e1483720f019aabb31aeae327161b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T115314466631Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78082fb5cea41b20cefb475310c0dd3b96c8f70174aa818f834a1cde2c18c3c5 +size 3919726 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T115216292984Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T115216292984Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9914228348dbc88045a79e4b87972477431daa58 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T115216292984Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457977344, + "active_bytes_current_delta": 37226522112, + "active_bytes_peak": 79934799360, + "active_bytes_peak_delta": 44468463104, + "after": { + "active_bytes_current": 72457977344, + "active_bytes_peak": 79934799360, + "allocated_bytes_current": 72457977344, + "allocated_bytes_peak": 79582477824, + "device": 7, + "inactive_split_bytes_current": 2808807936, + "inactive_split_bytes_peak": 5796429312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 96353648640, + "reserved_bytes_peak": 96353648640, + "segment_count_current": 509, + "segment_count_peak": 509, + "timestamp_utc": "2026-08-01T11:52:16.292432Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457977344, + "allocated_bytes_current_delta": 37226522112, + "allocated_bytes_peak": 79582477824, + "allocated_bytes_peak_delta": 44116141568, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:51:07.374633Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2808807936, + "inactive_split_bytes_current_delta": 2675988992, + "inactive_split_bytes_peak": 5796429312, + "inactive_split_bytes_peak_delta": 5636043264, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 96353648640, + "reserved_bytes_current_delta": 57109643264, + "reserved_bytes_peak": 96353648640, + "reserved_bytes_peak_delta": 57109643264, + "segment_count_current": 509, + "segment_count_current_delta": 438, + "segment_count_peak": 509, + "segment_count_peak_delta": 438, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T115216292984Z.pickle", + "timestamp_utc": "2026-08-01T11:52:16.406683Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T115216292984Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T115216292984Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..23f802a6e081260095f0c4979c5120a595ba302c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T115216292984Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02523db12b2f8dc7835d13bafbbd2f78e83b4a6a1f82d59c12cbe2e5de23d361 +size 5949662 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T115217414927Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T115217414927Z.json new file mode 100644 index 0000000000000000000000000000000000000000..41093de23956e058f1e98c2bdeafcf19b9b21041 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T115217414927Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457977344, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103575081472, + "active_bytes_peak_delta": 23640282112, + "after": { + "active_bytes_current": 72457977344, + "active_bytes_peak": 103575081472, + "allocated_bytes_current": 72457977344, + "allocated_bytes_peak": 103222759936, + "device": 7, + "inactive_split_bytes_current": 3282764288, + "inactive_split_bytes_peak": 5796429312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T11:52:17.414528Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457977344, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103222759936, + "allocated_bytes_peak_delta": 23640282112, + "before": { + "active_bytes_current": 72457977344, + "active_bytes_peak": 79934799360, + "allocated_bytes_current": 72457977344, + "allocated_bytes_peak": 79582477824, + "device": 7, + "inactive_split_bytes_current": 2808807936, + "inactive_split_bytes_peak": 5796429312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 96353648640, + "reserved_bytes_peak": 96353648640, + "segment_count_current": 509, + "segment_count_peak": 509, + "timestamp_utc": "2026-08-01T11:52:16.423709Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3282764288, + "inactive_split_bytes_current_delta": 473956352, + "inactive_split_bytes_peak": 5796429312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106380132352, + "reserved_bytes_current_delta": 10026483712, + "reserved_bytes_peak": 106380132352, + "reserved_bytes_peak_delta": 10026483712, + "segment_count_current": 580, + "segment_count_current_delta": 71, + "segment_count_peak": 580, + "segment_count_peak_delta": 71, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T115217414927Z.pickle", + "timestamp_utc": "2026-08-01T11:52:17.526671Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T115217414927Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T115217414927Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d63e8e01d93e0deb42d2bf6685fe1673baedcc11 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T115217414927Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7b42977cee40d13aee68071aa09562f598a278f184c6964ea384a7a1134eb6f +size 6582699 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T115218477903Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T115218477903Z.json new file mode 100644 index 0000000000000000000000000000000000000000..110b181a67cf446bfaf8e9f55859e68ff25f262e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T115218477903Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457977344, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103575081472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457977344, + "active_bytes_peak": 103575081472, + "allocated_bytes_current": 72457977344, + "allocated_bytes_peak": 103222759936, + "device": 7, + "inactive_split_bytes_current": 3253404160, + "inactive_split_bytes_peak": 5796429312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T11:52:18.477504Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457977344, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103222759936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457977344, + "active_bytes_peak": 103575081472, + "allocated_bytes_current": 72457977344, + "allocated_bytes_peak": 103222759936, + "device": 7, + "inactive_split_bytes_current": 3282764288, + "inactive_split_bytes_peak": 5796429312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T11:52:17.541127Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3253404160, + "inactive_split_bytes_current_delta": -29360128, + "inactive_split_bytes_peak": 5796429312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106380132352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106380132352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T115218477903Z.pickle", + "timestamp_utc": "2026-08-01T11:52:18.584792Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T115218477903Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T115218477903Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ae13ba1e2749d7346b1505d43fece0e599a280b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T115218477903Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38e07f9bd77d2921d0d26ecb6b7038506fd2a6716084a7cf96b34b9cb39bbc83 +size 5940167 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T115221500156Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T115221500156Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a220c75601b65ce75f24956e1994f973c845d540 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T115221500156Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475938304, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 160837370368, + "active_bytes_peak_delta": 57262288896, + "after": { + "active_bytes_current": 72475938304, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 72475938304, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_peak": 11156115968, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 175175106560, + "reserved_bytes_peak": 175175106560, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:21.499600Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475938304, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 103225303040, + "allocated_bytes_peak_delta": 2543104, + "before": { + "active_bytes_current": 72457977344, + "active_bytes_peak": 103575081472, + "allocated_bytes_current": 72457977344, + "allocated_bytes_peak": 103222759936, + "device": 7, + "inactive_split_bytes_current": 3253404160, + "inactive_split_bytes_peak": 5796429312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106380132352, + "reserved_bytes_peak": 106380132352, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T11:52:18.598484Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_current_delta": 4239257600, + "inactive_split_bytes_peak": 11156115968, + "inactive_split_bytes_peak_delta": 5359686656, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 175175106560, + "reserved_bytes_current_delta": 68794974208, + "reserved_bytes_peak": 175175106560, + "reserved_bytes_peak_delta": 68794974208, + "segment_count_current": 907, + "segment_count_current_delta": 327, + "segment_count_peak": 907, + "segment_count_peak_delta": 327, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T115221500156Z.pickle", + "timestamp_utc": "2026-08-01T11:52:21.628616Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T115221500156Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T115221500156Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..989c25013620a8d9e7c0c09bad48be73f97a2e0c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T115221500156Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd3fbf5d1d43e36ac3c0f6fa27146002d19aec0092708368df8ad92babd1b28e +size 6955527 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T115222449267Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T115222449267Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6dacefa86c640e982bf9403182ad8a471c0977da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T115222449267Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475938304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160837370368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475938304, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 72475938304, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_peak": 11156115968, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 175175106560, + "reserved_bytes_peak": 175175106560, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:22.448803Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475938304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103225303040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475938304, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 72475938304, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_peak": 11156115968, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 175175106560, + "reserved_bytes_peak": 175175106560, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:21.647194Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11156115968, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 175175106560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175175106560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 907, + "segment_count_current_delta": 0, + "segment_count_peak": 907, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T115222449267Z.pickle", + "timestamp_utc": "2026-08-01T11:52:22.455676Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T115222449267Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T115222449267Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b571fc482fcddb9f3d42aecf3f27fa26a731e66a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T115222449267Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da01598e2bc9f1dbf027e5cfb906dc43a7e05cd8d03130836e024c26b155b3e0 +size 248585 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T115223256743Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T115223256743Z.json new file mode 100644 index 0000000000000000000000000000000000000000..93097d493afcec12db8990e960384d122f3b3ecd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T115223256743Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475938304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160837370368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475938304, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 72475938304, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_peak": 11156115968, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 175175106560, + "reserved_bytes_peak": 175175106560, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:23.256317Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475938304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103225303040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475938304, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 72475938304, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_peak": 11156115968, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 175175106560, + "reserved_bytes_peak": 175175106560, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:22.456618Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11156115968, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 175175106560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175175106560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 907, + "segment_count_current_delta": 0, + "segment_count_peak": 907, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T115223256743Z.pickle", + "timestamp_utc": "2026-08-01T11:52:23.263026Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T115223256743Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T115223256743Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f4d70d06d7f87f9bd6c50f3bf294247cd770db6a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T115223256743Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0c0020312159147ddde2f3eef57d4b957fdb21e641c190aed7f48374c3dba87 +size 248585 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T115224063304Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T115224063304Z.json new file mode 100644 index 0000000000000000000000000000000000000000..562b23e7d9d1e473646ee98d4db6a53ff45cd066 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T115224063304Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475938304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160837370368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475938304, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 72475938304, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_peak": 11156115968, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 175175106560, + "reserved_bytes_peak": 175175106560, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:24.062906Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475938304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103225303040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475938304, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 72475938304, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_peak": 11156115968, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 175175106560, + "reserved_bytes_peak": 175175106560, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:23.263958Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11156115968, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 175175106560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175175106560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 907, + "segment_count_current_delta": 0, + "segment_count_peak": 907, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T115224063304Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.069234Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T115224063304Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T115224063304Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..277a99a5a5362be5a943104139cbd631af21e6bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T115224063304Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95fbfd41ebb080d796c25a8a74d6608cea7062df47d0c90893bb9b35091632a7 +size 248585 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T115224870159Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T115224870159Z.json new file mode 100644 index 0000000000000000000000000000000000000000..afa77fd3bbd0033af67ad4fe7a8bdca887a7880e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T115224870159Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475938304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160837370368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475938304, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 72475938304, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_peak": 11156115968, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 175175106560, + "reserved_bytes_peak": 175175106560, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:24.869754Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475938304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103225303040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475938304, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 72475938304, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_peak": 11156115968, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 175175106560, + "reserved_bytes_peak": 175175106560, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:24.070149Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11156115968, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 175175106560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175175106560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 907, + "segment_count_current_delta": 0, + "segment_count_peak": 907, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T115224870159Z.pickle", + "timestamp_utc": "2026-08-01T11:52:24.876108Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T115224870159Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T115224870159Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8f82c42bf30ffc08186588f8d5a2f1e7d4f79094 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T115224870159Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:455a78d5a03173894621fc06bda6de665c858d60a446a28f83d773ce8f60fec0 +size 248585 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T115225678980Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T115225678980Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fefe34fe6729250ee7c0355016d282fb9862f5e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T115225678980Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475938304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 160837370368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475938304, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 72475938304, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_peak": 11156115968, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 175175106560, + "reserved_bytes_peak": 175175106560, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:25.678583Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475938304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103225303040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475938304, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 72475938304, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_peak": 11156115968, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 175175106560, + "reserved_bytes_peak": 175175106560, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:24.877055Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11156115968, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 175175106560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175175106560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 907, + "segment_count_current_delta": 0, + "segment_count_peak": 907, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T115225678980Z.pickle", + "timestamp_utc": "2026-08-01T11:52:25.684861Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T115225678980Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T115225678980Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b0f8183f87f147a3079f2d335fc3b25e76560081 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T115225678980Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c9d5756dc22e3839cab9d493ee9d1b34ee2b846dd9d3b74bf11fd55e91d3249 +size 248585 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T115314466631Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T115314466631Z.json new file mode 100644 index 0000000000000000000000000000000000000000..54545922b01c74e310eeef6d5adc75b4cf03f26b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T115314466631Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61218657792, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 160837370368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61218657792, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 61218657792, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 3505744384, + "inactive_split_bytes_peak": 11156115968, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 176632627200, + "reserved_bytes_peak": 176632627200, + "segment_count_current": 918, + "segment_count_peak": 918, + "timestamp_utc": "2026-08-01T11:53:14.466158Z", + "world_size": 8 + }, + "allocated_bytes_current": 61218657792, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 103225303040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475938304, + "active_bytes_peak": 160837370368, + "allocated_bytes_current": 72475938304, + "allocated_bytes_peak": 103225303040, + "device": 7, + "inactive_split_bytes_current": 7492661760, + "inactive_split_bytes_peak": 11156115968, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 175175106560, + "reserved_bytes_peak": 175175106560, + "segment_count_current": 907, + "segment_count_peak": 907, + "timestamp_utc": "2026-08-01T11:52:25.685794Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3505744384, + "inactive_split_bytes_current_delta": -3986917376, + "inactive_split_bytes_peak": 11156115968, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 176632627200, + "reserved_bytes_current_delta": 1457520640, + "reserved_bytes_peak": 176632627200, + "reserved_bytes_peak_delta": 1457520640, + "segment_count_current": 918, + "segment_count_current_delta": 11, + "segment_count_peak": 918, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T115314466631Z.pickle", + "timestamp_utc": "2026-08-01T11:53:14.547888Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T115314466631Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T115314466631Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3efbd25adda9d70dc59b345160c425bd9bd1837 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T115314466631Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:266c4c9f951850e4564d5663e7df9d50418f3d2aadfba51112276c5de8b5e4f8 +size 4060073 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..bb44623da1120d57dad9edc52ee8cac875468473 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 2048 +capacity: 1.25 +expert_dispatch_overlap: off +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6004 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 230 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T115613180238Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T115613180238Z.json new file mode 100644 index 0000000000000000000000000000000000000000..12dfea0c2a7a6c62175545aa8a02c18d45a3f52f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T115613180238Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870811648, + "active_bytes_current_delta": 31639356416, + "active_bytes_peak": 80937462784, + "active_bytes_peak_delta": 45471126528, + "after": { + "active_bytes_current": 66870811648, + "active_bytes_peak": 80937462784, + "allocated_bytes_current": 66870811648, + "allocated_bytes_peak": 80937462784, + "device": 0, + "inactive_split_bytes_current": 5971665920, + "inactive_split_bytes_peak": 7013109760, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 95772737536, + "reserved_bytes_peak": 95772737536, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T11:56:13.179757Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870811648, + "allocated_bytes_current_delta": 31639356416, + "allocated_bytes_peak": 80937462784, + "allocated_bytes_peak_delta": 45471126528, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T11:55:03.169695Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5971665920, + "inactive_split_bytes_current_delta": 5836749824, + "inactive_split_bytes_peak": 7013109760, + "inactive_split_bytes_peak_delta": 6852477952, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 95772737536, + "reserved_bytes_current_delta": 56526635008, + "reserved_bytes_peak": 95772737536, + "reserved_bytes_peak_delta": 56526635008, + "segment_count_current": 503, + "segment_count_current_delta": 431, + "segment_count_peak": 503, + "segment_count_peak_delta": 431, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T115613180238Z.pickle", + "timestamp_utc": "2026-08-01T11:56:13.292218Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T115613180238Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T115613180238Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7083e7ad9e7722a90f1cbf0a257c476865350fbf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T115613180238Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c27d0f9ae6ff82e2e841ede01faa291b82e14be06e4d966aa82ad56572d7930 +size 5386105 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T115614374632Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T115614374632Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69e1c790f259a7e18ffac9a1918a425296d5536c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T115614374632Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870811648, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104564000768, + "active_bytes_peak_delta": 23626537984, + "after": { + "active_bytes_current": 66870811648, + "active_bytes_peak": 104564000768, + "allocated_bytes_current": 66870811648, + "allocated_bytes_peak": 104564000768, + "device": 0, + "inactive_split_bytes_current": 5422212096, + "inactive_split_bytes_peak": 8080544768, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106396909568, + "reserved_bytes_peak": 106396909568, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.374222Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870811648, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104564000768, + "allocated_bytes_peak_delta": 23626537984, + "before": { + "active_bytes_current": 66870811648, + "active_bytes_peak": 80937462784, + "allocated_bytes_current": 66870811648, + "allocated_bytes_peak": 80937462784, + "device": 0, + "inactive_split_bytes_current": 5971665920, + "inactive_split_bytes_peak": 7013109760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 95772737536, + "reserved_bytes_peak": 95772737536, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T11:56:13.311075Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5422212096, + "inactive_split_bytes_current_delta": -549453824, + "inactive_split_bytes_peak": 8080544768, + "inactive_split_bytes_peak_delta": 1067435008, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106396909568, + "reserved_bytes_current_delta": 10624172032, + "reserved_bytes_peak": 106396909568, + "reserved_bytes_peak_delta": 10624172032, + "segment_count_current": 560, + "segment_count_current_delta": 57, + "segment_count_peak": 560, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T115614374632Z.pickle", + "timestamp_utc": "2026-08-01T11:56:14.481827Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T115614374632Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T115614374632Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c334f0747a0d62b547ebff6075b986c3824b3642 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T115614374632Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ecffd176e6430728fe1d97bc75a1a7d8a3d3b29e21d9da085e9690d9456613a +size 5571310 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T115615619455Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T115615619455Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c61741129d583ad83a61897d3041826fb7d055e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T115615619455Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870811648, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104564000768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66870811648, + "active_bytes_peak": 104564000768, + "allocated_bytes_current": 66870811648, + "allocated_bytes_peak": 104564000768, + "device": 0, + "inactive_split_bytes_current": 6424650752, + "inactive_split_bytes_peak": 8080544768, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106396909568, + "reserved_bytes_peak": 106396909568, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.619015Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870811648, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104564000768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66870811648, + "active_bytes_peak": 104564000768, + "allocated_bytes_current": 66870811648, + "allocated_bytes_peak": 104564000768, + "device": 0, + "inactive_split_bytes_current": 5422212096, + "inactive_split_bytes_peak": 8080544768, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106396909568, + "reserved_bytes_peak": 106396909568, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.496983Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6424650752, + "inactive_split_bytes_current_delta": 1002438656, + "inactive_split_bytes_peak": 8080544768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106396909568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106396909568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T115615619455Z.pickle", + "timestamp_utc": "2026-08-01T11:56:15.725291Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T115615619455Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T115615619455Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..961e8e4d393eb883cd653740bfe96b4194490a70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T115615619455Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f958908204e71f9e6aa87b73c633da7d20f05516beb75832e68d1b24968aa20 +size 5572464 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T115618217502Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T115618217502Z.json new file mode 100644 index 0000000000000000000000000000000000000000..58fa77a23b2ba0bbbb7bc400fa51d07a44f5c949 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T115618217502Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888248320, + "active_bytes_current_delta": 17436672, + "active_bytes_peak": 122732789760, + "active_bytes_peak_delta": 18168788992, + "after": { + "active_bytes_current": 66888248320, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66888248320, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_peak": 11102893568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_peak": 137160032256, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:56:18.216930Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888248320, + "allocated_bytes_current_delta": 17436672, + "allocated_bytes_peak": 104592072192, + "allocated_bytes_peak_delta": 28071424, + "before": { + "active_bytes_current": 66870811648, + "active_bytes_peak": 104564000768, + "allocated_bytes_current": 66870811648, + "allocated_bytes_peak": 104564000768, + "device": 0, + "inactive_split_bytes_current": 6424650752, + "inactive_split_bytes_peak": 8080544768, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106396909568, + "reserved_bytes_peak": 106396909568, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.740261Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_current_delta": 313913344, + "inactive_split_bytes_peak": 11102893568, + "inactive_split_bytes_peak_delta": 3022348800, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_current_delta": 30763122688, + "reserved_bytes_peak": 137160032256, + "reserved_bytes_peak_delta": 30763122688, + "segment_count_current": 782, + "segment_count_current_delta": 222, + "segment_count_peak": 782, + "segment_count_peak_delta": 222, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T115618217502Z.pickle", + "timestamp_utc": "2026-08-01T11:56:18.336684Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T115618217502Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T115618217502Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f69e1d4183a95031ceda7393a40c9145fd3ec36d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T115618217502Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9acedd1dc590c32eb0dfb3bb738cbf918d6c9de71a35c30d0e886fc0f7286e34 +size 6309297 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T115619074591Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T115619074591Z.json new file mode 100644 index 0000000000000000000000000000000000000000..40bbe67ae14d5a56af9168ee133936547acb75c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T115619074591Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888248320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122732789760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888248320, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66888248320, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_peak": 11102893568, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_peak": 137160032256, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:56:19.074133Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888248320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104592072192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888248320, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66888248320, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_peak": 11102893568, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_peak": 137160032256, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:56:18.353172Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11102893568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137160032256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 782, + "segment_count_current_delta": 0, + "segment_count_peak": 782, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T115619074591Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.081758Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T115619074591Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T115619074591Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4523731b4dc6152d4e04276fe41b9817bb74002a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T115619074591Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27f985475f629ebe9b36dad69dc61e824648d708c647fe763ab40dc8c7f2019d +size 239603 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T115619784308Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T115619784308Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f5fdbc288d75dc450ac197365da83188de1bc80 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T115619784308Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888248320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122732789760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888248320, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66888248320, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_peak": 11102893568, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_peak": 137160032256, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:56:19.783883Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888248320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104592072192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888248320, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66888248320, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_peak": 11102893568, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_peak": 137160032256, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:56:19.082739Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11102893568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137160032256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 782, + "segment_count_current_delta": 0, + "segment_count_peak": 782, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T115619784308Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.791047Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T115619784308Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T115619784308Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e6389d5f7dfdb107b02e22feeacba176437e1445 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T115619784308Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7137ef19011de66c8190cd937c934cee1ffaa2fc95c325f1ce1a8f70d0b2d06c +size 249056 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T115620495562Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T115620495562Z.json new file mode 100644 index 0000000000000000000000000000000000000000..12e63fc1bff521aff16b3cf7d312a4e55d3ab67f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T115620495562Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888248320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122732789760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888248320, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66888248320, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_peak": 11102893568, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_peak": 137160032256, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:56:20.495166Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888248320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104592072192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888248320, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66888248320, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_peak": 11102893568, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_peak": 137160032256, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:56:19.792001Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11102893568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137160032256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 782, + "segment_count_current_delta": 0, + "segment_count_peak": 782, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T115620495562Z.pickle", + "timestamp_utc": "2026-08-01T11:56:20.501786Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T115620495562Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T115620495562Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..26a4c9dddf9c1f54263b53151648d6678a278543 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T115620495562Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1af18ae237b7c36f9c06dde5c1f4119d943b71a8a848de42fe02a771f1d2a23d +size 239603 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T115621203607Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T115621203607Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac1889f51b368842ed6d090614d9a0ee6e8d3e8b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T115621203607Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888248320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122732789760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888248320, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66888248320, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_peak": 11102893568, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_peak": 137160032256, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:56:21.203208Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888248320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104592072192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888248320, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66888248320, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_peak": 11102893568, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_peak": 137160032256, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:56:20.502761Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11102893568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137160032256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 782, + "segment_count_current_delta": 0, + "segment_count_peak": 782, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T115621203607Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.209633Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T115621203607Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T115621203607Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..efff0666a4679b1f5518be85cafe333c09176fe4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T115621203607Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75d53aa8506c77554f71f54d6fa7446b8223c7121c646c4029b27c3216dc7f33 +size 249056 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T115621915419Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T115621915419Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6100b30e5e7bf569ccf26aa3a98a075112c19f13 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T115621915419Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888248320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122732789760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888248320, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66888248320, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_peak": 11102893568, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_peak": 137160032256, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:56:21.915035Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888248320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104592072192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888248320, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66888248320, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_peak": 11102893568, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_peak": 137160032256, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:56:21.210582Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11102893568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137160032256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 782, + "segment_count_current_delta": 0, + "segment_count_peak": 782, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T115621915419Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.921426Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T115621915419Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T115621915419Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3c5d19701d37c991a85c00fcf3f78e59e1755fcd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T115621915419Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bd3f81ff739f073b0093495c32dedf5fe4fe9ef4f77758bfcd068a48a3c53c8 +size 239603 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T115709420816Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T115709420816Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f9889c44b34b39a41f8ddc9dccfb341613c27e7e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T115709420816Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854614016, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 122732789760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854614016, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66854614016, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 4614228992, + "inactive_split_bytes_peak": 11102893568, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137183100928, + "reserved_bytes_peak": 137183100928, + "segment_count_current": 784, + "segment_count_peak": 784, + "timestamp_utc": "2026-08-01T11:57:09.420341Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854614016, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104592072192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888248320, + "active_bytes_peak": 122732789760, + "allocated_bytes_current": 66888248320, + "allocated_bytes_peak": 104592072192, + "device": 0, + "inactive_split_bytes_current": 6738564096, + "inactive_split_bytes_peak": 11102893568, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 137160032256, + "reserved_bytes_peak": 137160032256, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:56:21.922378Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4614228992, + "inactive_split_bytes_current_delta": -2124335104, + "inactive_split_bytes_peak": 11102893568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137183100928, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 137183100928, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 784, + "segment_count_current_delta": 2, + "segment_count_peak": 784, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T115709420816Z.pickle", + "timestamp_utc": "2026-08-01T11:57:09.503840Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T115709420816Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T115709420816Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ebb691a637e17ad5c273df0de80d22e645a8337 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T115709420816Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6de2b44c3c6da483795d16263a49823c240456c2719d1ca65166165f23b22e80 +size 4172362 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T115613180229Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T115613180229Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0a7f5bf05e2e6e946d1b3c8b83c11ad6fd5ecc9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T115613180229Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 31639102464, + "active_bytes_peak": 80929376256, + "active_bytes_peak_delta": 45463040000, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80929376256, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80929376256, + "device": 1, + "inactive_split_bytes_current": 6259229696, + "inactive_split_bytes_peak": 7208666112, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 95770640384, + "reserved_bytes_peak": 95770640384, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T11:56:13.179551Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 31639102464, + "allocated_bytes_peak": 80929376256, + "allocated_bytes_peak_delta": 45463040000, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:55:03.169886Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6259229696, + "inactive_split_bytes_current_delta": 6126410752, + "inactive_split_bytes_peak": 7208666112, + "inactive_split_bytes_peak_delta": 7048280064, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 95770640384, + "reserved_bytes_current_delta": 56526635008, + "reserved_bytes_peak": 95770640384, + "reserved_bytes_peak_delta": 56526635008, + "segment_count_current": 502, + "segment_count_current_delta": 431, + "segment_count_peak": 502, + "segment_count_peak_delta": 431, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T115613180229Z.pickle", + "timestamp_utc": "2026-08-01T11:56:13.291099Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T115613180229Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T115613180229Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aeb82398bb9b4e7c6d23476796ebf87ae2c6b442 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T115613180229Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05aa7fb5626b90aa50358abb493e0594a72fb11e0571d23ba5eb2d64c90ef3b8 +size 5376745 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T115614374565Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T115614374565Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3a8c9f367d29b9c0518c2d1f5d4203d9a2c6141 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T115614374565Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104572923904, + "active_bytes_peak_delta": 23643547648, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572923904, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572923904, + "device": 1, + "inactive_split_bytes_current": 5854479360, + "inactive_split_bytes_peak": 8107049984, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.374133Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104572923904, + "allocated_bytes_peak_delta": 23643547648, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80929376256, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80929376256, + "device": 1, + "inactive_split_bytes_current": 6259229696, + "inactive_split_bytes_peak": 7208666112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 95770640384, + "reserved_bytes_peak": 95770640384, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T11:56:13.310960Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5854479360, + "inactive_split_bytes_current_delta": -404750336, + "inactive_split_bytes_peak": 8107049984, + "inactive_split_bytes_peak_delta": 898383872, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106552098816, + "reserved_bytes_current_delta": 10781458432, + "reserved_bytes_peak": 106552098816, + "reserved_bytes_peak_delta": 10781458432, + "segment_count_current": 560, + "segment_count_current_delta": 58, + "segment_count_peak": 560, + "segment_count_peak_delta": 58, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T115614374565Z.pickle", + "timestamp_utc": "2026-08-01T11:56:14.479007Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T115614374565Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T115614374565Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a3aa7204ac67f513ddbb6cf7315b0b183590a0bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T115614374565Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:270e4b504662fa2d45fd51729d7d2bb38abd97eac6ed4de9364d624eb5b1592e +size 5536601 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T115615619381Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T115615619381Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ee8fe3205ca201df900c7a7fc89d924b0df8264 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T115615619381Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104573259776, + "active_bytes_peak_delta": 335872, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104573259776, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104573259776, + "device": 1, + "inactive_split_bytes_current": 6412321792, + "inactive_split_bytes_peak": 8107049984, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.618943Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104573259776, + "allocated_bytes_peak_delta": 335872, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572923904, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572923904, + "device": 1, + "inactive_split_bytes_current": 5854479360, + "inactive_split_bytes_peak": 8107049984, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.494223Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6412321792, + "inactive_split_bytes_current_delta": 557842432, + "inactive_split_bytes_peak": 8107049984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106552098816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106552098816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T115615619381Z.pickle", + "timestamp_utc": "2026-08-01T11:56:15.722996Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T115615619381Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T115615619381Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2959d8d2244e5c4ec447b2514dd707bc8be74fc4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T115615619381Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed24de4a2eaf0d35735210306dee9d5f41a0b2eb84f70be992750b0efd4748df +size 5537780 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T115618217505Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T115618217505Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e1c48280f5b70404414de1025950bfcab9bf4d6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T115618217505Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 17436672, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 18168975360, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_peak": 11105545216, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137392816128, + "reserved_bytes_peak": 137392816128, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:18.216843Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 17436672, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 17378816, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104573259776, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104573259776, + "device": 1, + "inactive_split_bytes_current": 6412321792, + "inactive_split_bytes_peak": 8107049984, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.736267Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_current_delta": 349564928, + "inactive_split_bytes_peak": 11105545216, + "inactive_split_bytes_peak_delta": 2998495232, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137392816128, + "reserved_bytes_current_delta": 30840717312, + "reserved_bytes_peak": 137392816128, + "reserved_bytes_peak_delta": 30840717312, + "segment_count_current": 778, + "segment_count_current_delta": 218, + "segment_count_peak": 778, + "segment_count_peak_delta": 218, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T115618217505Z.pickle", + "timestamp_utc": "2026-08-01T11:56:18.345498Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T115618217505Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T115618217505Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0365d8afdcc5e5052337867447a1a25c8e096816 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T115618217505Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ed1011600233dd140ce90f4d4676eba79a23d671c95af97c6ca96022915c214 +size 5826652 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T115619074676Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T115619074676Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5df1ffb1e3b69b093b183c8ff320aa57da3290b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T115619074676Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_peak": 11105545216, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137392816128, + "reserved_bytes_peak": 137392816128, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:19.074126Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_peak": 11105545216, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 137392816128, + "reserved_bytes_peak": 137392816128, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:18.375391Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11105545216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137392816128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137392816128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 778, + "segment_count_current_delta": 0, + "segment_count_peak": 778, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T115619074676Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.082572Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T115619074676Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T115619074676Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ce0545425b41fa3080e93b6be98f69743170e9a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T115619074676Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a44e241a28319d900521e3f95f6ff53fdb3e241d2b711b25f18fb152a22d68dd +size 228688 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T115619784296Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T115619784296Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0fa1c4839351b1d13f780c0c41768a7127fb0a6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T115619784296Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_peak": 11105545216, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137392816128, + "reserved_bytes_peak": 137392816128, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:19.783860Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_peak": 11105545216, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 137392816128, + "reserved_bytes_peak": 137392816128, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:19.083611Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11105545216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137392816128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137392816128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 778, + "segment_count_current_delta": 0, + "segment_count_peak": 778, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T115619784296Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.791307Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T115619784296Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T115619784296Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2327d165d92f2884814c9c17f2ce41e29328eaf7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T115619784296Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:786dd3fcdb04a04472d9042c90ff63a99a4cb9d7f6ec9b6010e51eb474c2150c +size 228688 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T115620495551Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T115620495551Z.json new file mode 100644 index 0000000000000000000000000000000000000000..16c7a337edeb1f90b041a6ff8fdf678b0faa0e7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T115620495551Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_peak": 11105545216, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137392816128, + "reserved_bytes_peak": 137392816128, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:20.495145Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_peak": 11105545216, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 137392816128, + "reserved_bytes_peak": 137392816128, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:19.792311Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11105545216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137392816128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137392816128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 778, + "segment_count_current_delta": 0, + "segment_count_peak": 778, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T115620495551Z.pickle", + "timestamp_utc": "2026-08-01T11:56:20.502092Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T115620495551Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T115620495551Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0f5e642f45ed2c82fe2ca5102d7a34653fd28b2d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T115620495551Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae97d1e5f233d9af7087092a9fdaa68b36f4167bd51a68100da31e433fbc2194 +size 228688 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T115621203581Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T115621203581Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04e2711dd49ea2d4855cd45e1436516854081004 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T115621203581Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_peak": 11105545216, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137392816128, + "reserved_bytes_peak": 137392816128, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:21.203177Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_peak": 11105545216, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 137392816128, + "reserved_bytes_peak": 137392816128, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:20.503071Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11105545216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137392816128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137392816128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 778, + "segment_count_current_delta": 0, + "segment_count_peak": 778, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T115621203581Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.209975Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T115621203581Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T115621203581Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d1c4a0e8c14d44555b7940271005f1ca92d8036 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T115621203581Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:587e28c593ec29c3fc948714728f9c5e3b5feb90b66eb4056bb53f8cbaf60bea +size 228688 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T115621915602Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T115621915602Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd22e86c1e91cbda8de688fa41295bda1db827a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T115621915602Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_peak": 11105545216, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137392816128, + "reserved_bytes_peak": 137392816128, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:21.915091Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_peak": 11105545216, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 137392816128, + "reserved_bytes_peak": 137392816128, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:21.210937Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11105545216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137392816128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137392816128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 778, + "segment_count_current_delta": 0, + "segment_count_peak": 778, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T115621915602Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.922462Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T115621915602Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T115621915602Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44cb884e622959647a62a3816825c1a9fd7a0ee2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T115621915602Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:707cfac5e1d7dbedffc5c5046ba65894343b6cfa9cf223a363e2ac1f1f64ebea +size 228688 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T115709420869Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T115709420869Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a7d236e40e9ab32ac34fec4e1ae9c7291323bd4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T115709420869Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854360064, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854360064, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66854360064, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 3869993984, + "inactive_split_bytes_peak": 11105545216, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137415884800, + "reserved_bytes_peak": 137415884800, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:57:09.420372Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854360064, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 1, + "inactive_split_bytes_current": 6761886720, + "inactive_split_bytes_peak": 11105545216, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 137392816128, + "reserved_bytes_peak": 137392816128, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:21.923445Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3869993984, + "inactive_split_bytes_current_delta": -2891892736, + "inactive_split_bytes_peak": 11105545216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137415884800, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 137415884800, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 780, + "segment_count_current_delta": 2, + "segment_count_peak": 780, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T115709420869Z.pickle", + "timestamp_utc": "2026-08-01T11:57:09.508568Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T115709420869Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T115709420869Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a1704a444767b6df5d67450717522c7ce2bb030 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T115709420869Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ece0abb5dbce28d554f75758bcf7f7e2245fbbd895be6e330dabfe8b0998d612 +size 4021019 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T115613180357Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T115613180357Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb3489a64de7f972dcbe5866c65d7cdfd4c30d89 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T115613180357Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 31639102464, + "active_bytes_peak": 80932587008, + "active_bytes_peak_delta": 45466250752, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80932587008, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80932587008, + "device": 2, + "inactive_split_bytes_current": 6250841088, + "inactive_split_bytes_peak": 7862638592, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 95313461248, + "reserved_bytes_peak": 95313461248, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T11:56:13.179611Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 31639102464, + "allocated_bytes_peak": 80932587008, + "allocated_bytes_peak_delta": 45466250752, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:55:03.169851Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6250841088, + "inactive_split_bytes_current_delta": 6118022144, + "inactive_split_bytes_peak": 7862638592, + "inactive_split_bytes_peak_delta": 7702252544, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 95313461248, + "reserved_bytes_current_delta": 56069455872, + "reserved_bytes_peak": 95313461248, + "reserved_bytes_peak_delta": 56069455872, + "segment_count_current": 499, + "segment_count_current_delta": 428, + "segment_count_peak": 499, + "segment_count_peak_delta": 428, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T115613180357Z.pickle", + "timestamp_utc": "2026-08-01T11:56:13.290503Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T115613180357Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T115613180357Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d6f5b03a044527b1ba9b77c9a5b2e21171c66bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T115613180357Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dae2f1d2762ce5097fdeab9462a5c35af7c018fa4a52ec00bf2fc365de3fd181 +size 5378723 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T115614374551Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T115614374551Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e99046c1db5401f8188441081ec9399d6d722ee6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T115614374551Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104572241920, + "active_bytes_peak_delta": 23639654912, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572241920, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572241920, + "device": 2, + "inactive_split_bytes_current": 5602821120, + "inactive_split_bytes_peak": 8170650624, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106388520960, + "reserved_bytes_peak": 106388520960, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.374143Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104572241920, + "allocated_bytes_peak_delta": 23639654912, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80932587008, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80932587008, + "device": 2, + "inactive_split_bytes_current": 6250841088, + "inactive_split_bytes_peak": 7862638592, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 95313461248, + "reserved_bytes_peak": 95313461248, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T11:56:13.307043Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5602821120, + "inactive_split_bytes_current_delta": -648019968, + "inactive_split_bytes_peak": 8170650624, + "inactive_split_bytes_peak_delta": 308012032, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106388520960, + "reserved_bytes_current_delta": 11075059712, + "reserved_bytes_peak": 106388520960, + "reserved_bytes_peak_delta": 11075059712, + "segment_count_current": 560, + "segment_count_current_delta": 61, + "segment_count_peak": 560, + "segment_count_peak_delta": 61, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T115614374551Z.pickle", + "timestamp_utc": "2026-08-01T11:56:14.472793Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T115614374551Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T115614374551Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..beff4264412b2393cdbfb704d954179397042832 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T115614374551Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74e12f4adf1ca83d957ff533eb04bf4a4c8c86ccb3b03e5be4e0ba875b272aab +size 5533632 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T115615619351Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T115615619351Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a91a2c2e52673c6a856061844206b4a265448512 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T115615619351Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104572241920, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572241920, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572241920, + "device": 2, + "inactive_split_bytes_current": 6515082240, + "inactive_split_bytes_peak": 8170650624, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106388520960, + "reserved_bytes_peak": 106388520960, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.618923Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104572241920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572241920, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572241920, + "device": 2, + "inactive_split_bytes_current": 5602821120, + "inactive_split_bytes_peak": 8170650624, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106388520960, + "reserved_bytes_peak": 106388520960, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.484034Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6515082240, + "inactive_split_bytes_current_delta": 912261120, + "inactive_split_bytes_peak": 8170650624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106388520960, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106388520960, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T115615619351Z.pickle", + "timestamp_utc": "2026-08-01T11:56:15.725421Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T115615619351Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T115615619351Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..835067f100b812f4a985cc8d76c94e43d947d0bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T115615619351Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d86dcc7e3271437d43a2be710d2b01a60424f8c0add522c9ed44ad1323c8092 +size 5536138 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T115618217275Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T115618217275Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f4494e28cc15156e4899b8bc6e4a6ffaa782287a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T115618217275Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 17436672, + "active_bytes_peak": 122740006912, + "active_bytes_peak_delta": 18167764992, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_peak": 10896955392, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137470410752, + "reserved_bytes_peak": 137470410752, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:18.216773Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 17436672, + "allocated_bytes_peak": 104590310912, + "allocated_bytes_peak_delta": 18068992, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572241920, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572241920, + "device": 2, + "inactive_split_bytes_current": 6515082240, + "inactive_split_bytes_peak": 8170650624, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106388520960, + "reserved_bytes_peak": 106388520960, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.737691Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_current_delta": 464908288, + "inactive_split_bytes_peak": 10896955392, + "inactive_split_bytes_peak_delta": 2726304768, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137470410752, + "reserved_bytes_current_delta": 31081889792, + "reserved_bytes_peak": 137470410752, + "reserved_bytes_peak_delta": 31081889792, + "segment_count_current": 778, + "segment_count_current_delta": 218, + "segment_count_peak": 778, + "segment_count_peak_delta": 218, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T115618217275Z.pickle", + "timestamp_utc": "2026-08-01T11:56:18.333293Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T115618217275Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T115618217275Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4692d0d97f4196b03d5027ebe4d01ee8456c7dbc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T115618217275Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:097d8c6d0e8a690855c9c02f9bbfa7fec31f89acaca10df1662759c0a247c810 +size 5826331 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T115619074634Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T115619074634Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f170babc1e572b9b646d90a88b514677c147d59 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T115619074634Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122740006912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_peak": 10896955392, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137470410752, + "reserved_bytes_peak": 137470410752, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:19.074177Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590310912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_peak": 10896955392, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 137470410752, + "reserved_bytes_peak": 137470410752, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:18.346746Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10896955392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137470410752, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137470410752, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 778, + "segment_count_current_delta": 0, + "segment_count_peak": 778, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T115619074634Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.082811Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T115619074634Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T115619074634Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0831229644d800898699c6e057fd1fcc71814465 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T115619074634Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a39b07dcb4979680f2b7776db3fc5e4a2807cfa5c2394df285345bcdb9f07321 +size 228856 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T115619784136Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T115619784136Z.json new file mode 100644 index 0000000000000000000000000000000000000000..25cce01d9e938dd07f2b27323b7da757221525af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T115619784136Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122740006912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_peak": 10896955392, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137470410752, + "reserved_bytes_peak": 137470410752, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:19.783750Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590310912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_peak": 10896955392, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 137470410752, + "reserved_bytes_peak": 137470410752, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:19.083801Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10896955392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137470410752, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137470410752, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 778, + "segment_count_current_delta": 0, + "segment_count_peak": 778, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T115619784136Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.790386Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T115619784136Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T115619784136Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f908782b98864d36cafbf01fd7bb2d37662cdb8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T115619784136Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1af1a1ef985b68ffd4590e6018ee9af565a960eae2f2a03db24fd1fa974bc01c +size 228856 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T115620495431Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T115620495431Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab4196a032ef008a9ff9491a6965ad0b111407fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T115620495431Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122740006912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_peak": 10896955392, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137470410752, + "reserved_bytes_peak": 137470410752, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:20.495044Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590310912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_peak": 10896955392, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 137470410752, + "reserved_bytes_peak": 137470410752, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:19.791322Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10896955392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137470410752, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137470410752, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 778, + "segment_count_current_delta": 0, + "segment_count_peak": 778, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T115620495431Z.pickle", + "timestamp_utc": "2026-08-01T11:56:20.501766Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T115620495431Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T115620495431Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7023744bce021fbe1f8221a6eaa35a1adca52bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T115620495431Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b79acd1bf1a288f3f66b3eac07f57aaba504e53a5fa8691e458cb5dba525d4dc +size 228887 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T115621203446Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T115621203446Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad20deac9a0db2e2e7535ab5c0d38a48eab5dbf9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T115621203446Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122740006912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_peak": 10896955392, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137470410752, + "reserved_bytes_peak": 137470410752, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:21.203061Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590310912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_peak": 10896955392, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 137470410752, + "reserved_bytes_peak": 137470410752, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:20.502727Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10896955392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137470410752, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137470410752, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 778, + "segment_count_current_delta": 0, + "segment_count_peak": 778, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T115621203446Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.209244Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T115621203446Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T115621203446Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..78120d7377c8fa44b39221688b1e22203d391b04 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T115621203446Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bcf71ab777c2f7fdebec0eb25ca27d41ed4d8b4d7e2903e9f34e3f270672c82 +size 228856 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T115621915296Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T115621915296Z.json new file mode 100644 index 0000000000000000000000000000000000000000..37be70944617ae2bdfbf1a146f724d1be201af00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T115621915296Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122740006912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_peak": 10896955392, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137470410752, + "reserved_bytes_peak": 137470410752, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:21.914916Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590310912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_peak": 10896955392, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 137470410752, + "reserved_bytes_peak": 137470410752, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:21.210187Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10896955392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137470410752, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137470410752, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 778, + "segment_count_current_delta": 0, + "segment_count_peak": 778, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T115621915296Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.921036Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T115621915296Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T115621915296Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6af7b50ff61c3fb077c99c76139f81987d7019a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T115621915296Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b5c19e2f15705f1d46867d0499de02bbff67948202d63df72955743ad0e0624 +size 228887 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T115709420799Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T115709420799Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9819d7dbb3025ee819da67c07bc57843b8625099 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T115709420799Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854360064, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 122740006912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854360064, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66854360064, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 3972754432, + "inactive_split_bytes_peak": 10896955392, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137493479424, + "reserved_bytes_peak": 137493479424, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:57:09.420325Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854360064, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104590310912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122740006912, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590310912, + "device": 2, + "inactive_split_bytes_current": 6979990528, + "inactive_split_bytes_peak": 10896955392, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 137470410752, + "reserved_bytes_peak": 137470410752, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:56:21.921966Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3972754432, + "inactive_split_bytes_current_delta": -3007236096, + "inactive_split_bytes_peak": 10896955392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137493479424, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 137493479424, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 780, + "segment_count_current_delta": 2, + "segment_count_peak": 780, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T115709420799Z.pickle", + "timestamp_utc": "2026-08-01T11:57:09.511100Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T115709420799Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T115709420799Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5af64f1fc03e319aea8f05c64bc91458ca2f1304 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T115709420799Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b569c930b59919dc189101d445268a032052e792db319e7bec39f6c4cb4d6bc5 +size 4021046 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T115613180311Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T115613180311Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c6b01b7fef6bd37ac60f9e2c29a73babc6b3d01 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T115613180311Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 31639102464, + "active_bytes_peak": 80929376256, + "active_bytes_peak_delta": 45463040000, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80929376256, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80929376256, + "device": 3, + "inactive_split_bytes_current": 6259229696, + "inactive_split_bytes_peak": 7090996224, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 95770640384, + "reserved_bytes_peak": 95770640384, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T11:56:13.179603Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 31639102464, + "allocated_bytes_peak": 80929376256, + "allocated_bytes_peak_delta": 45463040000, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:55:03.169902Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6259229696, + "inactive_split_bytes_current_delta": 6126410752, + "inactive_split_bytes_peak": 7090996224, + "inactive_split_bytes_peak_delta": 6930610176, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 95770640384, + "reserved_bytes_current_delta": 56526635008, + "reserved_bytes_peak": 95770640384, + "reserved_bytes_peak_delta": 56526635008, + "segment_count_current": 502, + "segment_count_current_delta": 431, + "segment_count_peak": 502, + "segment_count_peak_delta": 431, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T115613180311Z.pickle", + "timestamp_utc": "2026-08-01T11:56:13.289493Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T115613180311Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T115613180311Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a0bf35db76fec4b8d353c1f5e36e0e0f935e1e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T115613180311Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6166705432d8448e2082bb29933d28e408383f29b71edddb7c056ade12cb3a6 +size 5376652 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T115614374517Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T115614374517Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e1c6dfa44d5797062c89419aca94054911946b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T115614374517Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104572923904, + "active_bytes_peak_delta": 23643547648, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572923904, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572923904, + "device": 3, + "inactive_split_bytes_current": 5854479360, + "inactive_split_bytes_peak": 7957161984, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.374114Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104572923904, + "allocated_bytes_peak_delta": 23643547648, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80929376256, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80929376256, + "device": 3, + "inactive_split_bytes_current": 6259229696, + "inactive_split_bytes_peak": 7090996224, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 95770640384, + "reserved_bytes_peak": 95770640384, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T11:56:13.306178Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5854479360, + "inactive_split_bytes_current_delta": -404750336, + "inactive_split_bytes_peak": 7957161984, + "inactive_split_bytes_peak_delta": 866165760, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106552098816, + "reserved_bytes_current_delta": 10781458432, + "reserved_bytes_peak": 106552098816, + "reserved_bytes_peak_delta": 10781458432, + "segment_count_current": 560, + "segment_count_current_delta": 58, + "segment_count_peak": 560, + "segment_count_peak_delta": 58, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T115614374517Z.pickle", + "timestamp_utc": "2026-08-01T11:56:14.471253Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T115614374517Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T115614374517Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..57fa5f5270c57824c92ae2d3e87240ba10bc5f62 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T115614374517Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eeaa0aba475a49e40820bdddc859ba4649c99319a13bb39314262b749fa8270 +size 5533451 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T115615619382Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T115615619382Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ab9644196351b4d422c9b983b8d8cf28ca920eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T115615619382Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104573259776, + "active_bytes_peak_delta": 335872, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104573259776, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104573259776, + "device": 3, + "inactive_split_bytes_current": 6416516096, + "inactive_split_bytes_peak": 7957161984, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.618967Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104573259776, + "allocated_bytes_peak_delta": 335872, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572923904, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572923904, + "device": 3, + "inactive_split_bytes_current": 5854479360, + "inactive_split_bytes_peak": 7957161984, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.482433Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6416516096, + "inactive_split_bytes_current_delta": 562036736, + "inactive_split_bytes_peak": 7957161984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106552098816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106552098816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T115615619382Z.pickle", + "timestamp_utc": "2026-08-01T11:56:15.731755Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T115615619382Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T115615619382Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..922ed7434e935b526c520586b5a9fcaf250c894f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T115615619382Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f3f596397da83b808315822c4e88a5cd75fba7977c39233d1bec1a0c82ea43a +size 5534786 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T115618217419Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T115618217419Z.json new file mode 100644 index 0000000000000000000000000000000000000000..98a83cb079a70f43af7ea59cfda15d6f9d823024 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T115618217419Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 17436672, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 18168975360, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_peak": 11107642368, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:18.216814Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 17436672, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 17378816, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104573259776, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104573259776, + "device": 3, + "inactive_split_bytes_current": 6416516096, + "inactive_split_bytes_peak": 7957161984, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.747310Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_current_delta": 347467776, + "inactive_split_bytes_peak": 11107642368, + "inactive_split_bytes_peak_delta": 3150480384, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137397010432, + "reserved_bytes_current_delta": 30844911616, + "reserved_bytes_peak": 137397010432, + "reserved_bytes_peak_delta": 30844911616, + "segment_count_current": 780, + "segment_count_current_delta": 220, + "segment_count_peak": 780, + "segment_count_peak_delta": 220, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T115618217419Z.pickle", + "timestamp_utc": "2026-08-01T11:56:18.331487Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T115618217419Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T115618217419Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cbd4e66e311d29181f2770f74001fca09c527a3e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T115618217419Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b0e0dffd033e50863b43870fd1c9813cf1cb8f63a8ddd8c6da70205395458b4 +size 5826899 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T115619074443Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T115619074443Z.json new file mode 100644 index 0000000000000000000000000000000000000000..caac1ae92d921e42193ef512777ef21f153063c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T115619074443Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_peak": 11107642368, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:19.073998Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_peak": 11107642368, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:18.344981Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11107642368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137397010432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137397010432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 780, + "segment_count_current_delta": 0, + "segment_count_peak": 780, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T115619074443Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.080588Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T115619074443Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T115619074443Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..622cce4e65e996443112e9f545f11ad2ed51ea4a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T115619074443Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aafe9c9b7e6d34da9395a6b30b1158dba0ae69ccde7305fcf176c7a5d7da6b0c +size 228933 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T115619784182Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T115619784182Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31da0824b47e3e5851f04cdbd85158afe5fd21f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T115619784182Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_peak": 11107642368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:19.783781Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_peak": 11107642368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:19.081536Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11107642368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137397010432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137397010432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 780, + "segment_count_current_delta": 0, + "segment_count_peak": 780, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T115619784182Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.790215Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T115619784182Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T115619784182Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1dd1bde594927f7798d403fa8929791d0ddb9a28 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T115619784182Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02bb4984b34fad341a72aa8d7eb9c71e752e1195e5de58704e06e297f3a7f80c +size 228933 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T115620495447Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T115620495447Z.json new file mode 100644 index 0000000000000000000000000000000000000000..73ab724906d4e4f3344066d38bbfaf6c87984acd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T115620495447Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_peak": 11107642368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:20.495044Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_peak": 11107642368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:19.791157Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11107642368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137397010432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137397010432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 780, + "segment_count_current_delta": 0, + "segment_count_peak": 780, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T115620495447Z.pickle", + "timestamp_utc": "2026-08-01T11:56:20.501299Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T115620495447Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T115620495447Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b01ef0ec8eb457abaa90aa019dbe44edacbc788c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T115620495447Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fd12424945f1276fa7f039b1f95d681edb83f9ae63881d40d1ff6693e61ebe4 +size 228933 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T115621203466Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T115621203466Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6d9ef220cd3cf530df9e2e27968d7550fa780613 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T115621203466Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_peak": 11107642368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:21.203073Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_peak": 11107642368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:20.502231Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11107642368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137397010432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137397010432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 780, + "segment_count_current_delta": 0, + "segment_count_peak": 780, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T115621203466Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.209282Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T115621203466Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T115621203466Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9aaa0dc9a0dc5f50fc53d2a75b37db06e095ebae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T115621203466Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2700c7790e9961e1b17e0ae84adb54398ed59b2ccef7e0cba099f449969db69b +size 228933 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T115621915329Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T115621915329Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ee1cb78b791750ae4ca950f4c38d900c5b509bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T115621915329Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_peak": 11107642368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:21.914940Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_peak": 11107642368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:21.210207Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11107642368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137397010432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137397010432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 780, + "segment_count_current_delta": 0, + "segment_count_peak": 780, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T115621915329Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.921171Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T115621915329Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T115621915329Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..67c35a5b52184fff668592f934d36798255ba880 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T115621915329Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbce14075a05d9936359e95cd635e62a577c503d2d57d21c102f7c0b8046e2f8 +size 228933 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T115709420799Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T115709420799Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6e94a6487c68e7225a4cdca21e6b0c5342d0a5a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T115709420799Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854360064, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 122742235136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854360064, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66854360064, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 3872091136, + "inactive_split_bytes_peak": 11107642368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137420079104, + "reserved_bytes_peak": 137420079104, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:57:09.420325Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854360064, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104590638592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122742235136, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104590638592, + "device": 3, + "inactive_split_bytes_current": 6763983872, + "inactive_split_bytes_peak": 11107642368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:21.922084Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3872091136, + "inactive_split_bytes_current_delta": -2891892736, + "inactive_split_bytes_peak": 11107642368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137420079104, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 137420079104, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 782, + "segment_count_current_delta": 2, + "segment_count_peak": 782, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T115709420799Z.pickle", + "timestamp_utc": "2026-08-01T11:57:09.506540Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T115709420799Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T115709420799Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f139bf3b1890a6df707fea1d43643bfd383c9c4e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T115709420799Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:562084aadfbddaa0d5dc5c050ac9b73a486d04ec8590f258aa5a848c91df1713 +size 4021254 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T115613180115Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T115613180115Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f64f5dee9a0e2e4097d9b47f066859483c5e77c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T115613180115Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872461312, + "active_bytes_current_delta": 31640563712, + "active_bytes_peak": 80927737344, + "active_bytes_peak_delta": 45460958720, + "after": { + "active_bytes_current": 66872461312, + "active_bytes_peak": 80927737344, + "allocated_bytes_current": 66872461312, + "allocated_bytes_peak": 80927737344, + "device": 4, + "inactive_split_bytes_current": 5443631104, + "inactive_split_bytes_peak": 6669268992, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 95298781184, + "reserved_bytes_peak": 95298781184, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T11:56:13.179627Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872461312, + "allocated_bytes_current_delta": 31640563712, + "allocated_bytes_peak": 80927737344, + "allocated_bytes_peak_delta": 45460958720, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:55:03.169507Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5443631104, + "inactive_split_bytes_current_delta": 5311254528, + "inactive_split_bytes_peak": 6669268992, + "inactive_split_bytes_peak_delta": 6515859456, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 95298781184, + "reserved_bytes_current_delta": 56054775808, + "reserved_bytes_peak": 95298781184, + "reserved_bytes_peak_delta": 56054775808, + "segment_count_current": 499, + "segment_count_current_delta": 428, + "segment_count_peak": 499, + "segment_count_peak_delta": 428, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T115613180115Z.pickle", + "timestamp_utc": "2026-08-01T11:56:13.292916Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T115613180115Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T115613180115Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a3c4c1ac604aab2ced88ebce3ce9cd65d292dcd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T115613180115Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dcdfa7a8939cdb88d51d762f66a9e07f795c21ef1d3d733b2ce863eabad866b +size 5385469 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T115614374510Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T115614374510Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e76d30b2bdd28e118a248cce8a3d02932a9d7526 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T115614374510Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872461312, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104570507776, + "active_bytes_peak_delta": 23642770432, + "after": { + "active_bytes_current": 66872461312, + "active_bytes_peak": 104570507776, + "allocated_bytes_current": 66872461312, + "allocated_bytes_peak": 104570507776, + "device": 4, + "inactive_split_bytes_current": 5978404864, + "inactive_split_bytes_peak": 7155897344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106008936448, + "reserved_bytes_peak": 106008936448, + "segment_count_current": 553, + "segment_count_peak": 553, + "timestamp_utc": "2026-08-01T11:56:14.374097Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872461312, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104570507776, + "allocated_bytes_peak_delta": 23642770432, + "before": { + "active_bytes_current": 66872461312, + "active_bytes_peak": 80927737344, + "allocated_bytes_current": 66872461312, + "allocated_bytes_peak": 80927737344, + "device": 4, + "inactive_split_bytes_current": 5443631104, + "inactive_split_bytes_peak": 6669268992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 95298781184, + "reserved_bytes_peak": 95298781184, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T11:56:13.312790Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5978404864, + "inactive_split_bytes_current_delta": 534773760, + "inactive_split_bytes_peak": 7155897344, + "inactive_split_bytes_peak_delta": 486628352, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106008936448, + "reserved_bytes_current_delta": 10710155264, + "reserved_bytes_peak": 106008936448, + "reserved_bytes_peak_delta": 10710155264, + "segment_count_current": 553, + "segment_count_current_delta": 54, + "segment_count_peak": 553, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T115614374510Z.pickle", + "timestamp_utc": "2026-08-01T11:56:14.480631Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T115614374510Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T115614374510Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e11c8067d6790bb228f7e2f3f1dc31c8cce2a9cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T115614374510Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60c05ec17798cf9033345bc2b503a91e3d50fa8cd6ef7d6ab9405fce8dbbc6b6 +size 5570149 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T115615619351Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T115615619351Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d1f6d4df1aa8f3251a235db5fa53058824244061 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T115615619351Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872461312, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104570901504, + "active_bytes_peak_delta": 393728, + "after": { + "active_bytes_current": 66872461312, + "active_bytes_peak": 104570901504, + "allocated_bytes_current": 66872461312, + "allocated_bytes_peak": 104570901504, + "device": 4, + "inactive_split_bytes_current": 6150371328, + "inactive_split_bytes_peak": 7354245120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106422075392, + "reserved_bytes_peak": 106422075392, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T11:56:15.618915Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872461312, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104570901504, + "allocated_bytes_peak_delta": 393728, + "before": { + "active_bytes_current": 66872461312, + "active_bytes_peak": 104570507776, + "allocated_bytes_current": 66872461312, + "allocated_bytes_peak": 104570507776, + "device": 4, + "inactive_split_bytes_current": 5978404864, + "inactive_split_bytes_peak": 7155897344, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106008936448, + "reserved_bytes_peak": 106008936448, + "segment_count_current": 553, + "segment_count_peak": 553, + "timestamp_utc": "2026-08-01T11:56:14.495658Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6150371328, + "inactive_split_bytes_current_delta": 171966464, + "inactive_split_bytes_peak": 7354245120, + "inactive_split_bytes_peak_delta": 198347776, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106422075392, + "reserved_bytes_current_delta": 413138944, + "reserved_bytes_peak": 106422075392, + "reserved_bytes_peak_delta": 413138944, + "segment_count_current": 554, + "segment_count_current_delta": 1, + "segment_count_peak": 554, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T115615619351Z.pickle", + "timestamp_utc": "2026-08-01T11:56:15.735274Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T115615619351Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T115615619351Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e15ad2c9a03cd67b16ff15e9fb07335685a426f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T115615619351Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edfaf299d20b90ce3b3ff12ad0ef83bc3717adf1f24c0296452a42206927d93b +size 5574034 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T115618217559Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T115618217559Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2808232b1a99f8d384a4dc9028ca7003b7fd8e1d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T115618217559Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889897984, + "active_bytes_current_delta": 17436672, + "active_bytes_peak": 122739944448, + "active_bytes_peak_delta": 18169042944, + "after": { + "active_bytes_current": 66889897984, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66889897984, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_peak": 10812917248, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137147449344, + "reserved_bytes_peak": 137147449344, + "segment_count_current": 776, + "segment_count_peak": 776, + "timestamp_utc": "2026-08-01T11:56:18.216924Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889897984, + "allocated_bytes_current_delta": 17436672, + "allocated_bytes_peak": 104591690240, + "allocated_bytes_peak_delta": 20788736, + "before": { + "active_bytes_current": 66872461312, + "active_bytes_peak": 104570901504, + "allocated_bytes_current": 66872461312, + "allocated_bytes_peak": 104570901504, + "device": 4, + "inactive_split_bytes_current": 6150371328, + "inactive_split_bytes_peak": 7354245120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106422075392, + "reserved_bytes_peak": 106422075392, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T11:56:15.753087Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_current_delta": 689303552, + "inactive_split_bytes_peak": 10812917248, + "inactive_split_bytes_peak_delta": 3458672128, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137147449344, + "reserved_bytes_current_delta": 30725373952, + "reserved_bytes_peak": 137147449344, + "reserved_bytes_peak_delta": 30725373952, + "segment_count_current": 776, + "segment_count_current_delta": 222, + "segment_count_peak": 776, + "segment_count_peak_delta": 222, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T115618217559Z.pickle", + "timestamp_utc": "2026-08-01T11:56:18.338843Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T115618217559Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T115618217559Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e8a2ca9afd968ebf991fbd34af73326386099c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T115618217559Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52a51a0af5f14d1ac251503b71e04777b791ce0f77ad9be3dd11cef2f833f241 +size 5979859 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T115619074508Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T115619074508Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a6ad27a61f5a7b003de71fb21e641f7280926518 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T115619074508Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889897984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122739944448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889897984, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66889897984, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_peak": 10812917248, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137147449344, + "reserved_bytes_peak": 137147449344, + "segment_count_current": 776, + "segment_count_peak": 776, + "timestamp_utc": "2026-08-01T11:56:19.074007Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889897984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104591690240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889897984, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66889897984, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_peak": 10812917248, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137147449344, + "reserved_bytes_peak": 137147449344, + "segment_count_current": 776, + "segment_count_peak": 776, + "timestamp_utc": "2026-08-01T11:56:18.355516Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10812917248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137147449344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137147449344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 776, + "segment_count_current_delta": 0, + "segment_count_peak": 776, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T115619074508Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.081632Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T115619074508Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T115619074508Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c79ffd77addec7f8eb805aacaeda5ae87423a41f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T115619074508Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d862cc373b50b782684cabe977b2008a58f0f78f9134e359a16bb0f8f5f3873 +size 236452 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T115619784158Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T115619784158Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d0a54b93372c261a4551737862fc11302234e1c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T115619784158Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889897984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122739944448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889897984, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66889897984, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_peak": 10812917248, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137147449344, + "reserved_bytes_peak": 137147449344, + "segment_count_current": 776, + "segment_count_peak": 776, + "timestamp_utc": "2026-08-01T11:56:19.783742Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889897984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104591690240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889897984, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66889897984, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_peak": 10812917248, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137147449344, + "reserved_bytes_peak": 137147449344, + "segment_count_current": 776, + "segment_count_peak": 776, + "timestamp_utc": "2026-08-01T11:56:19.082710Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10812917248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137147449344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137147449344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 776, + "segment_count_current_delta": 0, + "segment_count_peak": 776, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T115619784158Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.790929Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T115619784158Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T115619784158Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ad02093399d3abfb785f02c9dbe7e1fbe94dac5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T115619784158Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:326e94480c07b6fb2821e1c5b8b275056af42889296ce8d9e722ec13d632de87 +size 236452 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T115620495419Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T115620495419Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4dcba45975d1b73a8f9ed0bd4f1a6dcff31bb85 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T115620495419Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889897984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122739944448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889897984, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66889897984, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_peak": 10812917248, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137147449344, + "reserved_bytes_peak": 137147449344, + "segment_count_current": 776, + "segment_count_peak": 776, + "timestamp_utc": "2026-08-01T11:56:20.495011Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889897984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104591690240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889897984, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66889897984, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_peak": 10812917248, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137147449344, + "reserved_bytes_peak": 137147449344, + "segment_count_current": 776, + "segment_count_peak": 776, + "timestamp_utc": "2026-08-01T11:56:19.791881Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10812917248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137147449344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137147449344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 776, + "segment_count_current_delta": 0, + "segment_count_peak": 776, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T115620495419Z.pickle", + "timestamp_utc": "2026-08-01T11:56:20.501730Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T115620495419Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T115620495419Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6fe64257ce5f078e9ed45c44c3feee56b2fb8213 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T115620495419Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c97255ed274b4b4dc0b2e06c6b9b5203e06dfa49e1e674647d40674da0baa871 +size 236452 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T115621203462Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T115621203462Z.json new file mode 100644 index 0000000000000000000000000000000000000000..37776f74de37d87843213f6dff854ff56619afd6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T115621203462Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889897984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122739944448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889897984, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66889897984, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_peak": 10812917248, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137147449344, + "reserved_bytes_peak": 137147449344, + "segment_count_current": 776, + "segment_count_peak": 776, + "timestamp_utc": "2026-08-01T11:56:21.203055Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889897984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104591690240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889897984, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66889897984, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_peak": 10812917248, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137147449344, + "reserved_bytes_peak": 137147449344, + "segment_count_current": 776, + "segment_count_peak": 776, + "timestamp_utc": "2026-08-01T11:56:20.502701Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10812917248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137147449344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137147449344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 776, + "segment_count_current_delta": 0, + "segment_count_peak": 776, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T115621203462Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.209620Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T115621203462Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T115621203462Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c7259a9c24ad92b9ac9ccd9d20315542304886a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T115621203462Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c40f99cedfde08cf62a2037fec798f4fa20b2e94f06084ca20ab7c5581c70749 +size 236452 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T115621915301Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T115621915301Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b9dfec0f83e41bbac2793212c2c0b6a791c6f116 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T115621915301Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889897984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122739944448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889897984, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66889897984, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_peak": 10812917248, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137147449344, + "reserved_bytes_peak": 137147449344, + "segment_count_current": 776, + "segment_count_peak": 776, + "timestamp_utc": "2026-08-01T11:56:21.914899Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889897984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104591690240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889897984, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66889897984, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_peak": 10812917248, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137147449344, + "reserved_bytes_peak": 137147449344, + "segment_count_current": 776, + "segment_count_peak": 776, + "timestamp_utc": "2026-08-01T11:56:21.210551Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10812917248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137147449344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137147449344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 776, + "segment_count_current_delta": 0, + "segment_count_peak": 776, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T115621915301Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.921373Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T115621915301Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T115621915301Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b24c1056f2daff91e6db9fb57a0b63bb72a7d79f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T115621915301Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a05db049f650281dc4c27ee9a68043c4e6346a8f5a9a02be860f50c3930088b +size 236452 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T115709420834Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T115709420834Z.json new file mode 100644 index 0000000000000000000000000000000000000000..061f801e332336062bd959e5a2d18862d8f3dddd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T115709420834Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66856263680, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 122739944448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66856263680, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66856263680, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 4300103680, + "inactive_split_bytes_peak": 10812917248, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137170518016, + "reserved_bytes_peak": 137170518016, + "segment_count_current": 778, + "segment_count_peak": 778, + "timestamp_utc": "2026-08-01T11:57:09.420352Z", + "world_size": 8 + }, + "allocated_bytes_current": 66856263680, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104591690240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889897984, + "active_bytes_peak": 122739944448, + "allocated_bytes_current": 66889897984, + "allocated_bytes_peak": 104591690240, + "device": 4, + "inactive_split_bytes_current": 6839674880, + "inactive_split_bytes_peak": 10812917248, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137147449344, + "reserved_bytes_peak": 137147449344, + "segment_count_current": 776, + "segment_count_peak": 776, + "timestamp_utc": "2026-08-01T11:56:21.922306Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4300103680, + "inactive_split_bytes_current_delta": -2539571200, + "inactive_split_bytes_peak": 10812917248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137170518016, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 137170518016, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 778, + "segment_count_current_delta": 2, + "segment_count_peak": 778, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T115709420834Z.pickle", + "timestamp_utc": "2026-08-01T11:57:09.505875Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T115709420834Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T115709420834Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11a98de41315b6b8eb2467cb863dbe07051ade73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T115709420834Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1d2f94dddae9f32862b6236341346584aa9898f9158de7d12676f71c99e89f5 +size 4149565 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T115613180341Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T115613180341Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76caf0585888c7fd2868b2ca690c234ca5f9c14e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T115613180341Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 31639102464, + "active_bytes_peak": 80929376256, + "active_bytes_peak_delta": 45463040000, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80929376256, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80929376256, + "device": 5, + "inactive_split_bytes_current": 6263424000, + "inactive_split_bytes_peak": 7093093376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 95770640384, + "reserved_bytes_peak": 95770640384, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T11:56:13.179596Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 31639102464, + "allocated_bytes_peak": 80929376256, + "allocated_bytes_peak_delta": 45463040000, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:55:03.169848Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6263424000, + "inactive_split_bytes_current_delta": 6130605056, + "inactive_split_bytes_peak": 7093093376, + "inactive_split_bytes_peak_delta": 6932707328, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 95770640384, + "reserved_bytes_current_delta": 56526635008, + "reserved_bytes_peak": 95770640384, + "reserved_bytes_peak_delta": 56526635008, + "segment_count_current": 502, + "segment_count_current_delta": 431, + "segment_count_peak": 502, + "segment_count_peak_delta": 431, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T115613180341Z.pickle", + "timestamp_utc": "2026-08-01T11:56:13.292885Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T115613180341Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T115613180341Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..28901a2e2b989ff6d783edf88ad29a4899f78e05 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T115613180341Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4774d0128d2b918fbec850e17f5d83a05e0c37060a536670601ac77fda495e8f +size 5376544 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T115614374580Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T115614374580Z.json new file mode 100644 index 0000000000000000000000000000000000000000..166139b3a261000f2c65e624fd16af201636d06a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T115614374580Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104572923904, + "active_bytes_peak_delta": 23643547648, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572923904, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572923904, + "device": 5, + "inactive_split_bytes_current": 5829313536, + "inactive_split_bytes_peak": 7959259136, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.374130Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104572923904, + "allocated_bytes_peak_delta": 23643547648, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80929376256, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80929376256, + "device": 5, + "inactive_split_bytes_current": 6263424000, + "inactive_split_bytes_peak": 7093093376, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 95770640384, + "reserved_bytes_peak": 95770640384, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T11:56:13.312931Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5829313536, + "inactive_split_bytes_current_delta": -434110464, + "inactive_split_bytes_peak": 7959259136, + "inactive_split_bytes_peak_delta": 866165760, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106552098816, + "reserved_bytes_current_delta": 10781458432, + "reserved_bytes_peak": 106552098816, + "reserved_bytes_peak_delta": 10781458432, + "segment_count_current": 560, + "segment_count_current_delta": 58, + "segment_count_peak": 560, + "segment_count_peak_delta": 58, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T115614374580Z.pickle", + "timestamp_utc": "2026-08-01T11:56:14.482761Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T115614374580Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T115614374580Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b56c7ca518077ab7d7337544defafc6d6f51b2ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T115614374580Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f74165f68299a14a6aa7378f151df82ef71e7da4ce8fbbd3fe414283946f862e +size 5561316 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T115615619347Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T115615619347Z.json new file mode 100644 index 0000000000000000000000000000000000000000..21378635d0a7fd094104981f789e1e9fbd3d0bb0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T115615619347Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104573188096, + "active_bytes_peak_delta": 264192, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104573188096, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104573188096, + "device": 5, + "inactive_split_bytes_current": 6387155968, + "inactive_split_bytes_peak": 7959259136, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.618933Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104573188096, + "allocated_bytes_peak_delta": 264192, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572923904, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572923904, + "device": 5, + "inactive_split_bytes_current": 5829313536, + "inactive_split_bytes_peak": 7959259136, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.497794Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6387155968, + "inactive_split_bytes_current_delta": 557842432, + "inactive_split_bytes_peak": 7959259136, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106552098816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106552098816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T115615619347Z.pickle", + "timestamp_utc": "2026-08-01T11:56:15.726042Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T115615619347Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T115615619347Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..323d0f867be00801342dd3ac16fc30fa17fa99d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T115615619347Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87716d9a451bf0bc425eb5320a7c7ac2ba3c8249ad6c07bcde9099163bfe86a2 +size 5537483 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T115618217364Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T115618217364Z.json new file mode 100644 index 0000000000000000000000000000000000000000..02f77abdae864e3668196790ca8c5839c7e893fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T115618217364Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 17436672, + "active_bytes_peak": 122739482624, + "active_bytes_peak_delta": 18166294528, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_peak": 10894300160, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137357164544, + "reserved_bytes_peak": 137357164544, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:56:18.216815Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 17436672, + "allocated_bytes_peak": 104592604672, + "allocated_bytes_peak_delta": 19416576, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104573188096, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104573188096, + "device": 5, + "inactive_split_bytes_current": 6387155968, + "inactive_split_bytes_peak": 7959259136, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.740816Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_current_delta": 378925056, + "inactive_split_bytes_peak": 10894300160, + "inactive_split_bytes_peak_delta": 2935041024, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137357164544, + "reserved_bytes_current_delta": 30805065728, + "reserved_bytes_peak": 137357164544, + "reserved_bytes_peak_delta": 30805065728, + "segment_count_current": 779, + "segment_count_current_delta": 219, + "segment_count_peak": 779, + "segment_count_peak_delta": 219, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T115618217364Z.pickle", + "timestamp_utc": "2026-08-01T11:56:18.337324Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T115618217364Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T115618217364Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..31fbe233ca2d95e33153f48fd5ec18bd08542aeb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T115618217364Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30a2f7e00e237cec3b677524a9dd542771e9c04c47931a4cdd723b747c6311c6 +size 5826152 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T115619074517Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T115619074517Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e3151738df642fa84d2e3ca61a97d6842aaf4ef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T115619074517Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122739482624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_peak": 10894300160, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137357164544, + "reserved_bytes_peak": 137357164544, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:56:19.074043Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104592604672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_peak": 10894300160, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137357164544, + "reserved_bytes_peak": 137357164544, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:56:18.356458Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10894300160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137357164544, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137357164544, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 779, + "segment_count_current_delta": 0, + "segment_count_peak": 779, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T115619074517Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.081517Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T115619074517Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T115619074517Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..21dc8fc162b96f23863076c722d26a63d3e2874c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T115619074517Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f794bce554d5785f43cb93a0e68a66bbc94e0d501569bb03d0a6d531a8c77411 +size 228902 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T115619784209Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T115619784209Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d17928bde98fe2df323525a47331cbabe663bb81 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T115619784209Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122739482624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_peak": 10894300160, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137357164544, + "reserved_bytes_peak": 137357164544, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:56:19.783787Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104592604672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_peak": 10894300160, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137357164544, + "reserved_bytes_peak": 137357164544, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:56:19.082494Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10894300160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137357164544, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137357164544, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 779, + "segment_count_current_delta": 0, + "segment_count_peak": 779, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T115619784209Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.790848Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T115619784209Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T115619784209Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5f2ef8b8640d82135367d8f5a17c152839f932c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T115619784209Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c988323253e49508511c1b6739551a927eda61838d5bbffb825c63a405e1851 +size 228902 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T115620495468Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T115620495468Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f93fe16a792a9a58593576791d581df76c1dceb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T115620495468Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122739482624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_peak": 10894300160, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137357164544, + "reserved_bytes_peak": 137357164544, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:56:20.495070Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104592604672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_peak": 10894300160, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137357164544, + "reserved_bytes_peak": 137357164544, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:56:19.791813Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10894300160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137357164544, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137357164544, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 779, + "segment_count_current_delta": 0, + "segment_count_peak": 779, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T115620495468Z.pickle", + "timestamp_utc": "2026-08-01T11:56:20.501716Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T115620495468Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T115620495468Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f22a975812175d3cfdab7861b45230403fb9b641 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T115620495468Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ece1ca173ba23f1aed4ca6d775e36d881244a5d210df800081a034cfdc95ee9 +size 228902 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T115621203504Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T115621203504Z.json new file mode 100644 index 0000000000000000000000000000000000000000..58d69c2f184707693a0ee4d063907c2588c6dace --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T115621203504Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122739482624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_peak": 10894300160, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137357164544, + "reserved_bytes_peak": 137357164544, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:56:21.203103Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104592604672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_peak": 10894300160, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137357164544, + "reserved_bytes_peak": 137357164544, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:56:20.502643Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10894300160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137357164544, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137357164544, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 779, + "segment_count_current_delta": 0, + "segment_count_peak": 779, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T115621203504Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.209510Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T115621203504Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T115621203504Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b7d7be3754a64e2c754edacec385cedc5258ede4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T115621203504Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cc2c18b914ef9aff04d9c36613dfc534ff2525cef714debb86bce60240b0ac5 +size 228902 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T115621915327Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T115621915327Z.json new file mode 100644 index 0000000000000000000000000000000000000000..58492423566b1fdc83d0c91d9c6ef218226bab01 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T115621915327Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122739482624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_peak": 10894300160, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137357164544, + "reserved_bytes_peak": 137357164544, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:56:21.914942Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104592604672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_peak": 10894300160, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137357164544, + "reserved_bytes_peak": 137357164544, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:56:21.210435Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10894300160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137357164544, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137357164544, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 779, + "segment_count_current_delta": 0, + "segment_count_peak": 779, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T115621915327Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.921243Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T115621915327Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T115621915327Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a257c362989e2838240fdd9b1f682795d36c2d8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T115621915327Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bdc28833c57b0b08397cdee9e110d370ea9b03a11542f4d31fd63358595eaad +size 228902 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T115709420823Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T115709420823Z.json new file mode 100644 index 0000000000000000000000000000000000000000..68fca2085ecac30bec6bb23ee3d21d0ad06b49b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T115709420823Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854360064, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 122739482624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854360064, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66854360064, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 3991628800, + "inactive_split_bytes_peak": 10894300160, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137380233216, + "reserved_bytes_peak": 137380233216, + "segment_count_current": 781, + "segment_count_peak": 781, + "timestamp_utc": "2026-08-01T11:57:09.420337Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854360064, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104592604672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122739482624, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104592604672, + "device": 5, + "inactive_split_bytes_current": 6766081024, + "inactive_split_bytes_peak": 10894300160, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137357164544, + "reserved_bytes_peak": 137357164544, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:56:21.922159Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3991628800, + "inactive_split_bytes_current_delta": -2774452224, + "inactive_split_bytes_peak": 10894300160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137380233216, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 137380233216, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 781, + "segment_count_current_delta": 2, + "segment_count_peak": 781, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T115709420823Z.pickle", + "timestamp_utc": "2026-08-01T11:57:09.504182Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T115709420823Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T115709420823Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..010697cfebd0881003960dd1aa95cf260f7ed47f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T115709420823Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdb4478c2b3d97359f920331762e5d1ad4bd9a18d8dd5733504bb6dcfb560ec8 +size 4021287 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T115613180227Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T115613180227Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15cddec97b973378922690697b85dbd6d7214e84 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T115613180227Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 31639102464, + "active_bytes_peak": 80932587008, + "active_bytes_peak_delta": 45466250752, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80932587008, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80932587008, + "device": 6, + "inactive_split_bytes_current": 6250841088, + "inactive_split_bytes_peak": 7981930496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 95313461248, + "reserved_bytes_peak": 95313461248, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T11:56:13.179539Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 31639102464, + "allocated_bytes_peak": 80932587008, + "allocated_bytes_peak_delta": 45466250752, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:55:03.169835Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6250841088, + "inactive_split_bytes_current_delta": 6118022144, + "inactive_split_bytes_peak": 7981930496, + "inactive_split_bytes_peak_delta": 7821544448, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 95313461248, + "reserved_bytes_current_delta": 56069455872, + "reserved_bytes_peak": 95313461248, + "reserved_bytes_peak_delta": 56069455872, + "segment_count_current": 499, + "segment_count_current_delta": 428, + "segment_count_peak": 499, + "segment_count_peak_delta": 428, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T115613180227Z.pickle", + "timestamp_utc": "2026-08-01T11:56:13.289286Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T115613180227Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T115613180227Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ada6d1aadb67f52d6a1d9e5bdd3a0b3d71de0562 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T115613180227Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:875267e096d88c19b34fabb57a806dff3dc2b9a527df7630c7a90de877951dbc +size 5378577 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T115614374552Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T115614374552Z.json new file mode 100644 index 0000000000000000000000000000000000000000..757467279a8c304e835ad562fa19ab15ef4879f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T115614374552Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104572241920, + "active_bytes_peak_delta": 23639654912, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572241920, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572241920, + "device": 6, + "inactive_split_bytes_current": 5609112576, + "inactive_split_bytes_peak": 8170650624, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106388520960, + "reserved_bytes_peak": 106388520960, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.374138Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104572241920, + "allocated_bytes_peak_delta": 23639654912, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80932587008, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80932587008, + "device": 6, + "inactive_split_bytes_current": 6250841088, + "inactive_split_bytes_peak": 7981930496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 95313461248, + "reserved_bytes_peak": 95313461248, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T11:56:13.308730Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5609112576, + "inactive_split_bytes_current_delta": -641728512, + "inactive_split_bytes_peak": 8170650624, + "inactive_split_bytes_peak_delta": 188720128, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106388520960, + "reserved_bytes_current_delta": 11075059712, + "reserved_bytes_peak": 106388520960, + "reserved_bytes_peak_delta": 11075059712, + "segment_count_current": 560, + "segment_count_current_delta": 61, + "segment_count_peak": 560, + "segment_count_peak_delta": 61, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T115614374552Z.pickle", + "timestamp_utc": "2026-08-01T11:56:14.479892Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T115614374552Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T115614374552Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e43d982000566be6ae4c6a0ee2d45e1abd027eb8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T115614374552Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e4b55fa7a660de58ee90b3a528e110f742c906fd332a0f89740dd9326e74d8e +size 5536835 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T115615619574Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T115615619574Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1fee0213ebc57d20a1194bf391f286c27c9888ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T115615619574Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104572241920, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572241920, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572241920, + "device": 6, + "inactive_split_bytes_current": 6397641728, + "inactive_split_bytes_peak": 8170650624, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106388520960, + "reserved_bytes_peak": 106388520960, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.619073Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104572241920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572241920, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572241920, + "device": 6, + "inactive_split_bytes_current": 5609112576, + "inactive_split_bytes_peak": 8170650624, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106388520960, + "reserved_bytes_peak": 106388520960, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.495362Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6397641728, + "inactive_split_bytes_current_delta": 788529152, + "inactive_split_bytes_peak": 8170650624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106388520960, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106388520960, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T115615619574Z.pickle", + "timestamp_utc": "2026-08-01T11:56:15.739400Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T115615619574Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T115615619574Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba24acf73ec1f853961c6fa1a3d3e82e99230bf6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T115615619574Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:405ded5b016a58a5ce30d65c5157caa657fb6dcb0181f3564e35c71a97e5b96f +size 5537294 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T115618217524Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T115618217524Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d4405a3554359682eeac0f212471d03d6317f2ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T115618217524Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 17436672, + "active_bytes_peak": 122737975296, + "active_bytes_peak_delta": 18165733376, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_peak": 11315762176, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137025814528, + "reserved_bytes_peak": 137025814528, + "segment_count_current": 777, + "segment_count_peak": 777, + "timestamp_utc": "2026-08-01T11:56:18.216979Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 17436672, + "allocated_bytes_peak": 104588279296, + "allocated_bytes_peak_delta": 16037376, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104572241920, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104572241920, + "device": 6, + "inactive_split_bytes_current": 6397641728, + "inactive_split_bytes_peak": 8170650624, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106388520960, + "reserved_bytes_peak": 106388520960, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.759634Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_current_delta": 647360512, + "inactive_split_bytes_peak": 11315762176, + "inactive_split_bytes_peak_delta": 3145111552, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137025814528, + "reserved_bytes_current_delta": 30637293568, + "reserved_bytes_peak": 137025814528, + "reserved_bytes_peak_delta": 30637293568, + "segment_count_current": 777, + "segment_count_current_delta": 217, + "segment_count_peak": 777, + "segment_count_peak_delta": 217, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T115618217524Z.pickle", + "timestamp_utc": "2026-08-01T11:56:18.344052Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T115618217524Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T115618217524Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dfef8454a44c977f55c341c68e711e5e56faa08d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T115618217524Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dae66acb452eeb6d7f3ca4b42072fc52497cb625eb2e1eebe6399e8a4f6307b9 +size 5827215 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T115619074507Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T115619074507Z.json new file mode 100644 index 0000000000000000000000000000000000000000..66cc30fe9258beacedecdb77528094a3639747e7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T115619074507Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122737975296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_peak": 11315762176, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137025814528, + "reserved_bytes_peak": 137025814528, + "segment_count_current": 777, + "segment_count_peak": 777, + "timestamp_utc": "2026-08-01T11:56:19.074003Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104588279296, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_peak": 11315762176, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 137025814528, + "reserved_bytes_peak": 137025814528, + "segment_count_current": 777, + "segment_count_peak": 777, + "timestamp_utc": "2026-08-01T11:56:18.360487Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11315762176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137025814528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137025814528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 777, + "segment_count_current_delta": 0, + "segment_count_peak": 777, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T115619074507Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.081312Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T115619074507Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T115619074507Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cd299a9e2ffd06bfdf763cced4847132a6ae267d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T115619074507Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e62dff938e5c8e61948cbb692dd8ebf7ae1795403399f6f0e604d27beb334880 +size 229067 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T115619784170Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T115619784170Z.json new file mode 100644 index 0000000000000000000000000000000000000000..85878fc91e1acdba0aba1db9a9c35bfef924cdc3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T115619784170Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122737975296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_peak": 11315762176, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137025814528, + "reserved_bytes_peak": 137025814528, + "segment_count_current": 777, + "segment_count_peak": 777, + "timestamp_utc": "2026-08-01T11:56:19.783752Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104588279296, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_peak": 11315762176, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 137025814528, + "reserved_bytes_peak": 137025814528, + "segment_count_current": 777, + "segment_count_peak": 777, + "timestamp_utc": "2026-08-01T11:56:19.082268Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11315762176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137025814528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137025814528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 777, + "segment_count_current_delta": 0, + "segment_count_peak": 777, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T115619784170Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.790705Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T115619784170Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T115619784170Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..842a6104311f042483e660a6d5b8e6ce6a3222c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T115619784170Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f45d6d158f37179b1ac169e78c53555a2000e1aeeea4107a90afb55b98c944d +size 229067 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T115620495427Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T115620495427Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a98512b1545470d11bbb7148444273ac85d18da9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T115620495427Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122737975296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_peak": 11315762176, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137025814528, + "reserved_bytes_peak": 137025814528, + "segment_count_current": 777, + "segment_count_peak": 777, + "timestamp_utc": "2026-08-01T11:56:20.495024Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104588279296, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_peak": 11315762176, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 137025814528, + "reserved_bytes_peak": 137025814528, + "segment_count_current": 777, + "segment_count_peak": 777, + "timestamp_utc": "2026-08-01T11:56:19.791653Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11315762176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137025814528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137025814528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 777, + "segment_count_current_delta": 0, + "segment_count_peak": 777, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T115620495427Z.pickle", + "timestamp_utc": "2026-08-01T11:56:20.501573Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T115620495427Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T115620495427Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0f54d6eef253d969a1f5d29322b92bdeb41989f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T115620495427Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfee78c14f263914f3683c1d7ae6b8cb830ece4d15949c61a9c06d45c1e65f6c +size 229067 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T115621203515Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T115621203515Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7c7858f7258f452846fcb64f6f8bb176cb05fd0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T115621203515Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122737975296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_peak": 11315762176, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137025814528, + "reserved_bytes_peak": 137025814528, + "segment_count_current": 777, + "segment_count_peak": 777, + "timestamp_utc": "2026-08-01T11:56:21.203097Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104588279296, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_peak": 11315762176, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 137025814528, + "reserved_bytes_peak": 137025814528, + "segment_count_current": 777, + "segment_count_peak": 777, + "timestamp_utc": "2026-08-01T11:56:20.502488Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11315762176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137025814528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137025814528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 777, + "segment_count_current_delta": 0, + "segment_count_peak": 777, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T115621203515Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.209804Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T115621203515Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T115621203515Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1791e2f0ad5de69dda7241c1efeae7fc5614630f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T115621203515Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae6aa6b407c2d3f69387a53adad12daa491d0d0d2301053cb4e2792c746be0ac +size 229067 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T115621915294Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T115621915294Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0581cc32a7d49a180effa4da696093c7650fc22a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T115621915294Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994368, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122737975296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_peak": 11315762176, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137025814528, + "reserved_bytes_peak": 137025814528, + "segment_count_current": 777, + "segment_count_peak": 777, + "timestamp_utc": "2026-08-01T11:56:21.914902Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994368, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104588279296, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_peak": 11315762176, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 137025814528, + "reserved_bytes_peak": 137025814528, + "segment_count_current": 777, + "segment_count_peak": 777, + "timestamp_utc": "2026-08-01T11:56:21.210723Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11315762176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137025814528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137025814528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 777, + "segment_count_current_delta": 0, + "segment_count_peak": 777, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T115621915294Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.921743Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T115621915294Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T115621915294Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a743b5a212cf359e22d1b03a4fb3fa43690afc81 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T115621915294Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a70926e12e105b37a5860448dc12751fca332659502eb510df20eb603442b9c +size 229067 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T115709420817Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T115709420817Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b52f1923f11fbdc14140ed291568e0caa9a79fed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T115709420817Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854360064, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 122737975296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854360064, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66854360064, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 3863702528, + "inactive_split_bytes_peak": 11315762176, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137048883200, + "reserved_bytes_peak": 137048883200, + "segment_count_current": 779, + "segment_count_peak": 779, + "timestamp_utc": "2026-08-01T11:57:09.420337Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854360064, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104588279296, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994368, + "active_bytes_peak": 122737975296, + "allocated_bytes_current": 66887994368, + "allocated_bytes_peak": 104588279296, + "device": 6, + "inactive_split_bytes_current": 7045002240, + "inactive_split_bytes_peak": 11315762176, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 137025814528, + "reserved_bytes_peak": 137025814528, + "segment_count_current": 777, + "segment_count_peak": 777, + "timestamp_utc": "2026-08-01T11:56:21.922758Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3863702528, + "inactive_split_bytes_current_delta": -3181299712, + "inactive_split_bytes_peak": 11315762176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137048883200, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 137048883200, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 779, + "segment_count_current_delta": 2, + "segment_count_peak": 779, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T115709420817Z.pickle", + "timestamp_utc": "2026-08-01T11:57:09.514491Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T115709420817Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T115709420817Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c826884b011395bd105331e7bf82e7ee05d2b5e7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T115709420817Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38e652c0761798285663b3690db37b0d53fd400bea24df72e3b89a0daf11a1da +size 4021248 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T115613180293Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T115613180293Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f87bbc4753bbf1938e5a74a4b3322a0661e66ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T115613180293Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870558208, + "active_bytes_current_delta": 31639102976, + "active_bytes_peak": 80929376256, + "active_bytes_peak_delta": 45463040000, + "after": { + "active_bytes_current": 66870558208, + "active_bytes_peak": 80929376256, + "allocated_bytes_current": 66870558208, + "allocated_bytes_peak": 80929376256, + "device": 7, + "inactive_split_bytes_current": 6261326336, + "inactive_split_bytes_peak": 7093093376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 95770640384, + "reserved_bytes_peak": 95770640384, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T11:56:13.179564Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870558208, + "allocated_bytes_current_delta": 31639102976, + "allocated_bytes_peak": 80929376256, + "allocated_bytes_peak_delta": 45463040000, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:55:03.169922Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6261326336, + "inactive_split_bytes_current_delta": 6128507392, + "inactive_split_bytes_peak": 7093093376, + "inactive_split_bytes_peak_delta": 6932707328, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 95770640384, + "reserved_bytes_current_delta": 56526635008, + "reserved_bytes_peak": 95770640384, + "reserved_bytes_peak_delta": 56526635008, + "segment_count_current": 502, + "segment_count_current_delta": 431, + "segment_count_peak": 502, + "segment_count_peak_delta": 431, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T115613180293Z.pickle", + "timestamp_utc": "2026-08-01T11:56:13.290565Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T115613180293Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T115613180293Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2af037f12066cd1c08a1e036bbd2b5c3c08b3454 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T115613180293Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcb3342085387c32f54532648ef9dc8b0280d132e927aa4f565efbefaa65984a +size 5378159 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T115614374549Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T115614374549Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8219602e6112bc6cc0a4439e9781b18b199c93c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T115614374549Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870558208, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104572924416, + "active_bytes_peak_delta": 23643548160, + "after": { + "active_bytes_current": 66870558208, + "active_bytes_peak": 104572924416, + "allocated_bytes_current": 66870558208, + "allocated_bytes_peak": 104572924416, + "device": 7, + "inactive_split_bytes_current": 5856576000, + "inactive_split_bytes_peak": 8107049472, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.374126Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870558208, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104572924416, + "allocated_bytes_peak_delta": 23643548160, + "before": { + "active_bytes_current": 66870558208, + "active_bytes_peak": 80929376256, + "allocated_bytes_current": 66870558208, + "allocated_bytes_peak": 80929376256, + "device": 7, + "inactive_split_bytes_current": 6261326336, + "inactive_split_bytes_peak": 7093093376, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 95770640384, + "reserved_bytes_peak": 95770640384, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T11:56:13.310652Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5856576000, + "inactive_split_bytes_current_delta": -404750336, + "inactive_split_bytes_peak": 8107049472, + "inactive_split_bytes_peak_delta": 1013956096, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106552098816, + "reserved_bytes_current_delta": 10781458432, + "reserved_bytes_peak": 106552098816, + "reserved_bytes_peak_delta": 10781458432, + "segment_count_current": 560, + "segment_count_current_delta": 58, + "segment_count_peak": 560, + "segment_count_peak_delta": 58, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T115614374549Z.pickle", + "timestamp_utc": "2026-08-01T11:56:14.479921Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T115614374549Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T115614374549Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f77c573f8614356b35259e5f61c007ca3d85ad0b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T115614374549Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16d79ccf663852d2691a04d929113df298c27ffa4cb0541806ddc9be1bb758ef +size 5538744 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T115615619394Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T115615619394Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ce03f023140de9d48d189576f79c59d01000a57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T115615619394Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870558208, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104573260288, + "active_bytes_peak_delta": 335872, + "after": { + "active_bytes_current": 66870558208, + "active_bytes_peak": 104573260288, + "allocated_bytes_current": 66870558208, + "allocated_bytes_peak": 104573260288, + "device": 7, + "inactive_split_bytes_current": 6416515584, + "inactive_split_bytes_peak": 8107049472, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.618964Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870558208, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104573260288, + "allocated_bytes_peak_delta": 335872, + "before": { + "active_bytes_current": 66870558208, + "active_bytes_peak": 104572924416, + "allocated_bytes_current": 66870558208, + "allocated_bytes_peak": 104572924416, + "device": 7, + "inactive_split_bytes_current": 5856576000, + "inactive_split_bytes_peak": 8107049472, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:14.495317Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6416515584, + "inactive_split_bytes_current_delta": 559939584, + "inactive_split_bytes_peak": 8107049472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106552098816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106552098816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T115615619394Z.pickle", + "timestamp_utc": "2026-08-01T11:56:15.723717Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T115615619394Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T115615619394Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fed17fd0596db216ca194dd1e5b00c5a407ff593 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T115615619394Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c22306422313de54022d3f95245483ffa87da1ff49f3653b7dee976ba38e52ce +size 5539812 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T115618217443Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T115618217443Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a211e585dcd54d313476f6856075a4f5d4ccd7ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T115618217443Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994880, + "active_bytes_current_delta": 17436672, + "active_bytes_peak": 122742235648, + "active_bytes_peak_delta": 18168975360, + "after": { + "active_bytes_current": 66887994880, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66887994880, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_peak": 11103447552, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:18.216883Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994880, + "allocated_bytes_current_delta": 17436672, + "allocated_bytes_peak": 104590639104, + "allocated_bytes_peak_delta": 17378816, + "before": { + "active_bytes_current": 66870558208, + "active_bytes_peak": 104573260288, + "allocated_bytes_current": 66870558208, + "allocated_bytes_peak": 104573260288, + "device": 7, + "inactive_split_bytes_current": 6416515584, + "inactive_split_bytes_peak": 8107049472, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106552098816, + "reserved_bytes_peak": 106552098816, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T11:56:15.737095Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_current_delta": 349564928, + "inactive_split_bytes_peak": 11103447552, + "inactive_split_bytes_peak_delta": 2996398080, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137397010432, + "reserved_bytes_current_delta": 30844911616, + "reserved_bytes_peak": 137397010432, + "reserved_bytes_peak_delta": 30844911616, + "segment_count_current": 780, + "segment_count_current_delta": 220, + "segment_count_peak": 780, + "segment_count_peak_delta": 220, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T115618217443Z.pickle", + "timestamp_utc": "2026-08-01T11:56:18.336868Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T115618217443Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T115618217443Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11971aabd3304df67621ebfc3180f6d8d2f0072a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T115618217443Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c03717b0e304a5d6f2e81301aa78bbbdc57ccbd54e8a519f48ea6c719816f4a4 +size 5828514 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T115619074521Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T115619074521Z.json new file mode 100644 index 0000000000000000000000000000000000000000..663bad1c1bf30337bef410775abdb38f8c5b3162 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T115619074521Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994880, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66887994880, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_peak": 11103447552, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:19.074046Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590639104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994880, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66887994880, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_peak": 11103447552, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:18.352432Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11103447552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137397010432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137397010432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 780, + "segment_count_current_delta": 0, + "segment_count_peak": 780, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T115619074521Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.081126Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T115619074521Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T115619074521Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe4a7107ee3dd066027745ec78ed8e65f5fe72b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T115619074521Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fdf14cd21972c32d4d3f74adfaed73c76de37cfc1856eece90df4947662859d +size 230654 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T115619784235Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T115619784235Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0c3a0bc059d8f64d1a90ea2ee6aa89b9cc17a9bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T115619784235Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994880, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66887994880, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_peak": 11103447552, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:19.783820Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590639104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994880, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66887994880, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_peak": 11103447552, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:19.082066Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11103447552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137397010432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137397010432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 780, + "segment_count_current_delta": 0, + "segment_count_peak": 780, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T115619784235Z.pickle", + "timestamp_utc": "2026-08-01T11:56:19.790566Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T115619784235Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T115619784235Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e32f45fa18f64d33feb172590b0470ad1a248edf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T115619784235Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65b516070187bbbf1f906d6d149c7a763dd550e171ba91f4d8e34ecf70f7b599 +size 230623 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T115620495470Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T115620495470Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f9ff722f8005a6dcf1774b97be8bd30d50cf8ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T115620495470Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994880, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66887994880, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_peak": 11103447552, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:20.495075Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590639104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994880, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66887994880, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_peak": 11103447552, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:19.791492Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11103447552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137397010432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137397010432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 780, + "segment_count_current_delta": 0, + "segment_count_peak": 780, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T115620495470Z.pickle", + "timestamp_utc": "2026-08-01T11:56:20.501473Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T115620495470Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T115620495470Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d65642f434d7b112b777255c938b58880b8f1496 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T115620495470Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c78b21b3e7c12d3c3b23435e459f74f9d850e546ac5d854f498984c3108bc626 +size 230654 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T115621203605Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T115621203605Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b15637262027c4cdf4c1d8929759ff408adb977f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T115621203605Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994880, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66887994880, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_peak": 11103447552, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:21.203212Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590639104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994880, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66887994880, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_peak": 11103447552, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:20.502383Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11103447552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137397010432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137397010432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 780, + "segment_count_current_delta": 0, + "segment_count_peak": 780, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T115621203605Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.210004Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T115621203605Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T115621203605Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1cf8b00f43e35988033cd9804e56528920d652b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T115621203605Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e849d5dc4eb3dd26381af6c489d9b75f7c11e2bf32fff42e000284e65e0723c2 +size 230654 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T115621915377Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T115621915377Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f957d317790c131343c6fca7a04c3af3c37bddb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T115621915377Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887994880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122742235648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887994880, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66887994880, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_peak": 11103447552, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:21.914977Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887994880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104590639104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994880, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66887994880, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_peak": 11103447552, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:21.210936Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11103447552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137397010432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137397010432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 780, + "segment_count_current_delta": 0, + "segment_count_peak": 780, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T115621915377Z.pickle", + "timestamp_utc": "2026-08-01T11:56:21.921256Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T115621915377Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T115621915377Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..25c9208a02b507e669a2fb000b8ef052190aa9bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T115621915377Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:549fa2b09d53beee04ccf5e16357a0e146955c62ee14cf6faaec3e0c2b481c42 +size 230623 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T115709420818Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T115709420818Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d87f0a6bbfb505a8819f3f01832e64633a57ece0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T115709420818Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854360576, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 122742235648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854360576, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66854360576, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 3874187776, + "inactive_split_bytes_peak": 11103447552, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137420079104, + "reserved_bytes_peak": 137420079104, + "segment_count_current": 782, + "segment_count_peak": 782, + "timestamp_utc": "2026-08-01T11:57:09.420355Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854360576, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104590639104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887994880, + "active_bytes_peak": 122742235648, + "allocated_bytes_current": 66887994880, + "allocated_bytes_peak": 104590639104, + "device": 7, + "inactive_split_bytes_current": 6766080512, + "inactive_split_bytes_peak": 11103447552, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137397010432, + "reserved_bytes_peak": 137397010432, + "segment_count_current": 780, + "segment_count_peak": 780, + "timestamp_utc": "2026-08-01T11:56:21.922175Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3874187776, + "inactive_split_bytes_current_delta": -2891892736, + "inactive_split_bytes_peak": 11103447552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137420079104, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 137420079104, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 782, + "segment_count_current_delta": 2, + "segment_count_peak": 782, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T115709420818Z.pickle", + "timestamp_utc": "2026-08-01T11:57:09.508780Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T115709420818Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T115709420818Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc53ce83ec134661016640abed5f18add9b1b0fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T115709420818Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57873871089f1b313ba95252f24d31a2107380fc4d5f9f029aa506430646ced3 +size 4023008 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..dd66a07b3cd7da0fe64a5ce85c827fbdad59b63a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 2048 +capacity: 1.25 +expert_dispatch_overlap: off +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6005 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 231 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T120007789586Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T120007789586Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b9511cc5ec01d2782f5bb0476eae8d329046c3a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T120007789586Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72458329088, + "active_bytes_current_delta": 37226873856, + "active_bytes_peak": 80854277120, + "active_bytes_peak_delta": 45387940864, + "after": { + "active_bytes_current": 72458329088, + "active_bytes_peak": 80854277120, + "allocated_bytes_current": 72458329088, + "allocated_bytes_peak": 80854277120, + "device": 0, + "inactive_split_bytes_current": 6409266176, + "inactive_split_bytes_peak": 8848489472, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 91601502208, + "reserved_bytes_peak": 91601502208, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T12:00:07.789017Z", + "world_size": 8 + }, + "allocated_bytes_current": 72458329088, + "allocated_bytes_current_delta": 37226873856, + "allocated_bytes_peak": 80854277120, + "allocated_bytes_peak_delta": 45387940864, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T11:58:58.879731Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6409266176, + "inactive_split_bytes_current_delta": 6274350080, + "inactive_split_bytes_peak": 8848489472, + "inactive_split_bytes_peak_delta": 8687857664, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 91601502208, + "reserved_bytes_current_delta": 52355399680, + "reserved_bytes_peak": 91601502208, + "reserved_bytes_peak_delta": 52355399680, + "segment_count_current": 422, + "segment_count_current_delta": 350, + "segment_count_peak": 422, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T120007789586Z.pickle", + "timestamp_utc": "2026-08-01T12:00:07.903044Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T120007789586Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T120007789586Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b6af1fde472bda3e1332ba97c60bbd8d8af24c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T120007789586Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c05d6ee2284554c9b86db6e966509b8bc3df46ed384ad0e9de57b660876103cf +size 5932770 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T120008936907Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T120008936907Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f58ea671bbf5e15c9c87811e6438ed29b4f149fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T120008936907Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72458329088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104486704640, + "active_bytes_peak_delta": 23632427520, + "after": { + "active_bytes_current": 72458329088, + "active_bytes_peak": 104486704640, + "allocated_bytes_current": 72458329088, + "allocated_bytes_peak": 104486704640, + "device": 0, + "inactive_split_bytes_current": 7757734912, + "inactive_split_bytes_peak": 8848489472, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106585653248, + "reserved_bytes_peak": 106585653248, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T12:00:08.936502Z", + "world_size": 8 + }, + "allocated_bytes_current": 72458329088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104486704640, + "allocated_bytes_peak_delta": 23632427520, + "before": { + "active_bytes_current": 72458329088, + "active_bytes_peak": 80854277120, + "allocated_bytes_current": 72458329088, + "allocated_bytes_peak": 80854277120, + "device": 0, + "inactive_split_bytes_current": 6409266176, + "inactive_split_bytes_peak": 8848489472, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 91601502208, + "reserved_bytes_peak": 91601502208, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T12:00:07.918754Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7757734912, + "inactive_split_bytes_current_delta": 1348468736, + "inactive_split_bytes_peak": 8848489472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106585653248, + "reserved_bytes_current_delta": 14984151040, + "reserved_bytes_peak": 106585653248, + "reserved_bytes_peak_delta": 14984151040, + "segment_count_current": 490, + "segment_count_current_delta": 68, + "segment_count_peak": 490, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T120008936907Z.pickle", + "timestamp_utc": "2026-08-01T12:00:09.046077Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T120008936907Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T120008936907Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1000ff2b8eb29e3afc08039e8b70cada9e9b1ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T120008936907Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c676c3269e33048e06c987578bf83db8acb7b3377b73842f02db1fece3e57ed +size 5915325 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T120010015570Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T120010015570Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6a135fa887913afa96b3907c95061891ff777ad0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T120010015570Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72458329088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104486704640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72458329088, + "active_bytes_peak": 104486704640, + "allocated_bytes_current": 72458329088, + "allocated_bytes_peak": 104486704640, + "device": 0, + "inactive_split_bytes_current": 7229252608, + "inactive_split_bytes_peak": 8848489472, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106585653248, + "reserved_bytes_peak": 106585653248, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T12:00:10.015170Z", + "world_size": 8 + }, + "allocated_bytes_current": 72458329088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104486704640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72458329088, + "active_bytes_peak": 104486704640, + "allocated_bytes_current": 72458329088, + "allocated_bytes_peak": 104486704640, + "device": 0, + "inactive_split_bytes_current": 7757734912, + "inactive_split_bytes_peak": 8848489472, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106585653248, + "reserved_bytes_peak": 106585653248, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T12:00:09.058273Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7229252608, + "inactive_split_bytes_current_delta": -528482304, + "inactive_split_bytes_peak": 8848489472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106585653248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106585653248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T120010015570Z.pickle", + "timestamp_utc": "2026-08-01T12:00:10.122400Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T120010015570Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T120010015570Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d5dd585f1005a9704afe4c76595386810be046b4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T120010015570Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:754b7784dc9feb9ca25bba7fd969a84dd2662354055a1cb6c11500e501ef169a +size 5909783 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T120012827240Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T120012827240Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e142b141035a0cc6e8388b7a0e0abf7a8485f7e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T120012827240Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475372544, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 122654399488, + "active_bytes_peak_delta": 18167694848, + "after": { + "active_bytes_current": 72475372544, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 72475372544, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_peak": 15171281920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_peak": 137462022144, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:00:12.826724Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475372544, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104504440832, + "allocated_bytes_peak_delta": 17736192, + "before": { + "active_bytes_current": 72458329088, + "active_bytes_peak": 104486704640, + "allocated_bytes_current": 72458329088, + "allocated_bytes_peak": 104486704640, + "device": 0, + "inactive_split_bytes_current": 7229252608, + "inactive_split_bytes_peak": 8848489472, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106585653248, + "reserved_bytes_peak": 106585653248, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T12:00:10.134802Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_current_delta": 2709254144, + "inactive_split_bytes_peak": 15171281920, + "inactive_split_bytes_peak_delta": 6322792448, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_current_delta": 30876368896, + "reserved_bytes_peak": 137462022144, + "reserved_bytes_peak_delta": 30876368896, + "segment_count_current": 671, + "segment_count_current_delta": 181, + "segment_count_peak": 671, + "segment_count_peak_delta": 181, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T120012827240Z.pickle", + "timestamp_utc": "2026-08-01T12:00:12.948441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T120012827240Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T120012827240Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d7e3fb98df4a4171422429fdd30aa5692b6fb999 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T120012827240Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26208e1c7a46fb7c4252de87a28a161b18c23c6d81d0c819e5d2d86301f0422f +size 6323828 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T120013801113Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T120013801113Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36632025e39c2cc7e7994aad5ef8068747c7c339 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T120013801113Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475372544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122654399488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475372544, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 72475372544, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_peak": 15171281920, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_peak": 137462022144, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:00:13.800681Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475372544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104504440832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475372544, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 72475372544, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_peak": 15171281920, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_peak": 137462022144, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:00:12.964470Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15171281920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137462022144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 671, + "segment_count_current_delta": 0, + "segment_count_peak": 671, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T120013801113Z.pickle", + "timestamp_utc": "2026-08-01T12:00:13.805669Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T120013801113Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T120013801113Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fce6b353de3159683f5dcec585a75ee4e93201b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T120013801113Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00b3cf4dce5f7eb1496d394c483468e2aa10abb76a21b06f38f21ce283ebbd66 +size 172941 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T120014637187Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T120014637187Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06c29e79a6b8689f16d854fb503b31aeed4ebe37 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T120014637187Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475372544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122654399488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475372544, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 72475372544, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_peak": 15171281920, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_peak": 137462022144, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:00:14.636786Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475372544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104504440832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475372544, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 72475372544, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_peak": 15171281920, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_peak": 137462022144, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:00:13.806567Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15171281920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137462022144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 671, + "segment_count_current_delta": 0, + "segment_count_peak": 671, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T120014637187Z.pickle", + "timestamp_utc": "2026-08-01T12:00:14.641746Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T120014637187Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T120014637187Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dddcb0bd0b318f05c595fdb5ad0982a32cae3324 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T120014637187Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edf5262ae598625d74e7177cbd9f7662b106812dd99571d7ffd23b1a14bd7e5d +size 172941 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T120015472691Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T120015472691Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9af571528de31d52ed229ad9ab6a798a23bb8e34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T120015472691Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475372544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122654399488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475372544, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 72475372544, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_peak": 15171281920, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_peak": 137462022144, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:00:15.472308Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475372544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104504440832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475372544, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 72475372544, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_peak": 15171281920, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_peak": 137462022144, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:00:14.642619Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15171281920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137462022144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 671, + "segment_count_current_delta": 0, + "segment_count_peak": 671, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T120015472691Z.pickle", + "timestamp_utc": "2026-08-01T12:00:15.477087Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T120015472691Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T120015472691Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a8470e1df7084e479fdd4310f821dcc244e6e29 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T120015472691Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6545e0536890f62a569feb61977aeb87d79ef0f83f1a0b004869d6886748b1b7 +size 172941 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T120016308889Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T120016308889Z.json new file mode 100644 index 0000000000000000000000000000000000000000..84f29846e97033577fa06f2fd81976116928a419 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T120016308889Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475372544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122654399488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475372544, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 72475372544, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_peak": 15171281920, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_peak": 137462022144, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:00:16.308447Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475372544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104504440832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475372544, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 72475372544, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_peak": 15171281920, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_peak": 137462022144, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:00:15.477927Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15171281920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137462022144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 671, + "segment_count_current_delta": 0, + "segment_count_peak": 671, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T120016308889Z.pickle", + "timestamp_utc": "2026-08-01T12:00:16.313945Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T120016308889Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T120016308889Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..77ce54039a20e89a78812184f0fb8c26d7575c28 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T120016308889Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dc5b17f693c2b9491788f0c48603f4ce54e8495d8d1acac998f70bcde49e066 +size 172941 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T120017145682Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T120017145682Z.json new file mode 100644 index 0000000000000000000000000000000000000000..680d666a4ec4e85d33ecfe59803c19e89327b381 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T120017145682Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475372544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122654399488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475372544, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 72475372544, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_peak": 15171281920, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_peak": 137462022144, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:00:17.145278Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475372544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104504440832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475372544, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 72475372544, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_peak": 15171281920, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_peak": 137462022144, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:00:16.314813Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15171281920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137462022144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 671, + "segment_count_current_delta": 0, + "segment_count_peak": 671, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T120017145682Z.pickle", + "timestamp_utc": "2026-08-01T12:00:17.150678Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T120017145682Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T120017145682Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..821cd758d3cfd1684943f059632312e18a5327b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T120017145682Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b90f7e308c39e378229ae209cc1ad5829dbee979ebfd67aba7a05174f61988f +size 172941 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T120105675519Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T120105675519Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bb07820b086c7d7a3b50802f04877f24cf6ac328 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T120105675519Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61218092032, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 122654399488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61218092032, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 61218092032, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 7828540416, + "inactive_split_bytes_peak": 15171281920, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137898229760, + "reserved_bytes_peak": 137898229760, + "segment_count_current": 674, + "segment_count_peak": 674, + "timestamp_utc": "2026-08-01T12:01:05.675048Z", + "world_size": 8 + }, + "allocated_bytes_current": 61218092032, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104504440832, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475372544, + "active_bytes_peak": 122654399488, + "allocated_bytes_current": 72475372544, + "allocated_bytes_peak": 104504440832, + "device": 0, + "inactive_split_bytes_current": 9938506752, + "inactive_split_bytes_peak": 15171281920, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 137462022144, + "reserved_bytes_peak": 137462022144, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:00:17.151541Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7828540416, + "inactive_split_bytes_current_delta": -2109966336, + "inactive_split_bytes_peak": 15171281920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137898229760, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137898229760, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 674, + "segment_count_current_delta": 3, + "segment_count_peak": 674, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T120105675519Z.pickle", + "timestamp_utc": "2026-08-01T12:01:05.753837Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T120105675519Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T120105675519Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d2296d594806d437920aedd4e6a71c19d03d5e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T120105675519Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddb3505a57a01f0ba56b0873867bf57fcf03a0838d639886b82327cfd333d85e +size 3965240 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T120007789719Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T120007789719Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f76cba17c79657c0c80d7b236459ab7a976360f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T120007789719Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 80846497280, + "active_bytes_peak_delta": 45380161024, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 80846497280, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80846497280, + "device": 1, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_peak": 8821705728, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 91513421824, + "reserved_bytes_peak": 91513421824, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T12:00:07.789123Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 80846497280, + "allocated_bytes_peak_delta": 45380161024, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:58:58.880109Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_current_delta": 6378380288, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 8661319680, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 91513421824, + "reserved_bytes_current_delta": 52269416448, + "reserved_bytes_peak": 91513421824, + "reserved_bytes_peak_delta": 52269416448, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T120007789719Z.pickle", + "timestamp_utc": "2026-08-01T12:00:07.906457Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T120007789719Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T120007789719Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2aed7674e399fd62f2e673fde1291c9ea01ef807 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T120007789719Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c6d9ccb70d564971e108ca293921da68e03723714d16fb7c42b9657f8835b40 +size 5923786 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T120008936896Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T120008936896Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b37d3fdeabb8941cbd9c55f581caa42887ec2d95 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T120008936896Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104490119680, + "active_bytes_peak_delta": 23643622400, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 1, + "inactive_split_bytes_current": 6683165696, + "inactive_split_bytes_peak": 8821705728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:08.936489Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104490119680, + "allocated_bytes_peak_delta": 23643622400, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 80846497280, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80846497280, + "device": 1, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_peak": 8821705728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 91513421824, + "reserved_bytes_peak": 91513421824, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T12:00:07.924781Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6683165696, + "inactive_split_bytes_current_delta": 171966464, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106493378560, + "reserved_bytes_current_delta": 14979956736, + "reserved_bytes_peak": 106493378560, + "reserved_bytes_peak_delta": 14979956736, + "segment_count_current": 487, + "segment_count_current_delta": 67, + "segment_count_peak": 487, + "segment_count_peak_delta": 67, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T120008936896Z.pickle", + "timestamp_utc": "2026-08-01T12:00:09.041849Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T120008936896Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T120008936896Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..54af1f69d3fccde0bc7690d87f3e065d5297b9d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T120008936896Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c77f8d224f257f712cd01c4cbad6687b07029da04bfd2622ac1e4166137b56a7 +size 5894426 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T120010015530Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T120010015530Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9e5b3ec85d280feaea7c8ca3c48794831aebe47c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T120010015530Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104490119680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 1, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_peak": 8821705728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:10.015116Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104490119680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 1, + "inactive_split_bytes_current": 6683165696, + "inactive_split_bytes_peak": 8821705728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:09.055326Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_current_delta": 205520896, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106493378560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106493378560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T120010015530Z.pickle", + "timestamp_utc": "2026-08-01T12:00:10.122448Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T120010015530Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T120010015530Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..045df3f3cdf701f0dbee8da7eab74ba67beb3fbf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T120010015530Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0de18fa82860a681da6c1cf85624ec2027a4b6843a1630fbe2e522376235ff0 +size 5890277 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T120012827289Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T120012827289Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7be685ebac9d3e0ec724836e65047680ce8dc4d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T120012827289Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 17698816, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 18157308416, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:12.826717Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 17698816, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 15869440, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 1, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_peak": 8821705728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:10.136068Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 2207379456, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 5991446528, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 30802968576, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 30802968576, + "segment_count_current": 668, + "segment_count_current_delta": 181, + "segment_count_peak": 668, + "segment_count_peak_delta": 181, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T120012827289Z.pickle", + "timestamp_utc": "2026-08-01T12:00:12.947390Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T120012827289Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T120012827289Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a17e524fc536e4fd604ac7753d294715c7cc786 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T120012827289Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8209c7ba3e9aecd55d3bbe45fc4234486790be67ed8d932e410c26fd50bff20d +size 6168472 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T120013801147Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T120013801147Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f57cfa5a78604730c2b28a45f551c53c0fceb55 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T120013801147Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:13.800695Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:12.962429Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T120013801147Z.pickle", + "timestamp_utc": "2026-08-01T12:00:13.805848Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T120013801147Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T120013801147Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..89c8242b673a63adb2b5bcc60a3f6730483a2a64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T120013801147Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d97dfc3f224508a2bb534497d358e5a79ea1f62e24739fd99a41bc77895da5fd +size 168218 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T120014637174Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T120014637174Z.json new file mode 100644 index 0000000000000000000000000000000000000000..055ef05343c2dc8fcf8d148e8ce094fd1afc61b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T120014637174Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:14.636763Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:13.806744Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T120014637174Z.pickle", + "timestamp_utc": "2026-08-01T12:00:14.641825Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T120014637174Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T120014637174Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..24150f71f43ffd562be7ac91eb96263cc76cb0c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T120014637174Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:159ef02b5a28a822f09f8b7a1b5b65ea4d84a4bcaeeac7e00733f06b7f991bff +size 168218 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T120015472830Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T120015472830Z.json new file mode 100644 index 0000000000000000000000000000000000000000..08d31a3cf322c5824607b6e79e5726f9647e7b58 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T120015472830Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:15.472345Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:14.642712Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T120015472830Z.pickle", + "timestamp_utc": "2026-08-01T12:00:15.477658Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T120015472830Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T120015472830Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a7514f78988c37b1f65180773cdca1ee5771ad8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T120015472830Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ca7d61e0f0bf329d645c2e05feb809f8d1feb0ef211e11ee32cd6cff373c3f1 +size 168218 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T120016308864Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T120016308864Z.json new file mode 100644 index 0000000000000000000000000000000000000000..044b8a3adbb8a809303f3e0ba052efd5514df878 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T120016308864Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:16.308421Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:15.478538Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T120016308864Z.pickle", + "timestamp_utc": "2026-08-01T12:00:16.313723Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T120016308864Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T120016308864Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e05c1c892dcb65eda5c97c6e1070a92efec8bee2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T120016308864Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:979829449e7de182e4fd27aaebbaeefba71c22eecf4161c1538bede61280c0a5 +size 168218 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T120017145636Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T120017145636Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fecd64f76c6f6191df28f8b1a338cf01727afd07 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T120017145636Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:17.145246Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:16.314654Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T120017145636Z.pickle", + "timestamp_utc": "2026-08-01T12:00:17.150041Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T120017145636Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T120017145636Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29036dd146822c9c703183dbf0a0bb26bd54f79b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T120017145636Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3d7c7aaa6e1afcb3f08d7bf4466a97963336220302d05562008a08b5754b62b +size 168218 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T120105675528Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T120105675528Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b836da0437d3f430c517c3d9cdd50c9bb56b91af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T120105675528Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61217477632, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61217477632, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 61217477632, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 7105637376, + "inactive_split_bytes_peak": 14813152256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137732554752, + "reserved_bytes_peak": 137732554752, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:01:05.675053Z", + "world_size": 8 + }, + "allocated_bytes_current": 61217477632, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 1, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:17.150923Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7105637376, + "inactive_split_bytes_current_delta": -1990428672, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137732554752, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137732554752, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 671, + "segment_count_current_delta": 3, + "segment_count_peak": 671, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T120105675528Z.pickle", + "timestamp_utc": "2026-08-01T12:01:05.755410Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T120105675528Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T120105675528Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6bf0052e19f9adab5cef30641cdb6c2e9f110ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T120105675528Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04346364a4179a41598ed452be8d171faa5b645725709d171344447027752976 +size 3838307 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T120007789750Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T120007789750Z.json new file mode 100644 index 0000000000000000000000000000000000000000..81206a4bf995593c7b7466f4a2efe79ba53fb643 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T120007789750Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 80846497280, + "active_bytes_peak_delta": 45380161024, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 80846497280, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80846497280, + "device": 2, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_peak": 8821705728, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 91513421824, + "reserved_bytes_peak": 91513421824, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T12:00:07.789149Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 80846497280, + "allocated_bytes_peak_delta": 45380161024, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:58:58.880069Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_current_delta": 6378380288, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 8661319680, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 91513421824, + "reserved_bytes_current_delta": 52269416448, + "reserved_bytes_peak": 91513421824, + "reserved_bytes_peak_delta": 52269416448, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T120007789750Z.pickle", + "timestamp_utc": "2026-08-01T12:00:07.908986Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T120007789750Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T120007789750Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..32b6d1e261c12f047e13e437992d3c487c56c7b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T120007789750Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdef8d8fe431992807107c42a42b28ea2b8ab238511d2cd9672d285627541fad +size 5923791 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T120008936921Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T120008936921Z.json new file mode 100644 index 0000000000000000000000000000000000000000..26a91cce80c1ab4afe4ce53ae556f4265a9dbc52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T120008936921Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104490119680, + "active_bytes_peak_delta": 23643622400, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 2, + "inactive_split_bytes_current": 6685262848, + "inactive_split_bytes_peak": 8821705728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:08.936522Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104490119680, + "allocated_bytes_peak_delta": 23643622400, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 80846497280, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80846497280, + "device": 2, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_peak": 8821705728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 91513421824, + "reserved_bytes_peak": 91513421824, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T12:00:07.927928Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6685262848, + "inactive_split_bytes_current_delta": 174063616, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106493378560, + "reserved_bytes_current_delta": 14979956736, + "reserved_bytes_peak": 106493378560, + "reserved_bytes_peak_delta": 14979956736, + "segment_count_current": 487, + "segment_count_current_delta": 67, + "segment_count_peak": 487, + "segment_count_peak_delta": 67, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T120008936921Z.pickle", + "timestamp_utc": "2026-08-01T12:00:09.049187Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T120008936921Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T120008936921Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7319a30f23d6d46adfd2e5e128a3c6c88a6c4a1c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T120008936921Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae42601d74e6c186064f727de61defbce324872dd862b1a562dac9a7af1f06ba +size 5894429 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T120010015522Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T120010015522Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd31256a99340023d675488742d89fcac938bc9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T120010015522Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104490119680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 2, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_peak": 8821705728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:10.015128Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104490119680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 2, + "inactive_split_bytes_current": 6685262848, + "inactive_split_bytes_peak": 8821705728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:09.064803Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_current_delta": 203423744, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106493378560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106493378560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T120010015522Z.pickle", + "timestamp_utc": "2026-08-01T12:00:10.124316Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T120010015522Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T120010015522Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..97c2c5284865bfd8819f158f4a62bda3367fb508 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T120010015522Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56f32ee79c4efe53386428d19cb4e4eefd1d7eb727bd063ac2eda2216b31e91c +size 5890275 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T120012827310Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T120012827310Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7898227ba758bb325de2011177bc204be6ff21b4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T120012827310Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 17698816, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 18157308416, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:12.826767Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 17698816, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 15869440, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 2, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_peak": 8821705728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:10.139618Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 2207379456, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 5991446528, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 30802968576, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 30802968576, + "segment_count_current": 668, + "segment_count_current_delta": 181, + "segment_count_peak": 668, + "segment_count_peak_delta": 181, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T120012827310Z.pickle", + "timestamp_utc": "2026-08-01T12:00:12.951426Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T120012827310Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T120012827310Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..012b948761ddced1df5ccef24090f49a0478724c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T120012827310Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e5ccdb56346eeb608f24b23b2b6c722484df30ebdae29562686a58aedc9068d +size 6168472 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T120013801316Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T120013801316Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8dadb4c3b42391c53d9a1d1d967503e37f7baa85 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T120013801316Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:13.800754Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:12.969219Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T120013801316Z.pickle", + "timestamp_utc": "2026-08-01T12:00:13.806522Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T120013801316Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T120013801316Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c3ccb5d1101d3bffe2f9e8132e14c9c3d68d16f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T120013801316Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e43c6315834afaa11fbcddc9e6174a698ce393969bb051ab0a9688c320ab7e1 +size 168218 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T120014637224Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T120014637224Z.json new file mode 100644 index 0000000000000000000000000000000000000000..07934df160a7429504a95c01ec3ccf720248b757 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T120014637224Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:14.636818Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:13.807413Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T120014637224Z.pickle", + "timestamp_utc": "2026-08-01T12:00:14.641948Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T120014637224Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T120014637224Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5f04c6087befc387ea59b6fb9217612b77c24e5c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T120014637224Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98221776c697cd748d6b63d2fc8492a27bd2f34c43d3c7c2557427dada3d0313 +size 168218 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T120015472748Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T120015472748Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e603e5780f33b555ba9ca30d7ec9177a5e2936f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T120015472748Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:15.472351Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:14.642856Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T120015472748Z.pickle", + "timestamp_utc": "2026-08-01T12:00:15.477218Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T120015472748Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T120015472748Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3974f681bf4abe742b0f6a77d7b34531222b36ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T120015472748Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a65761ac8567dae32e4aa3f62b824fdeb9749d20b25dac789458bc273f2a992 +size 168218 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T120016309010Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T120016309010Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1fd5715fe3591e8c6b49db5b7a591f35d7a17286 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T120016309010Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:16.308518Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:15.478062Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T120016309010Z.pickle", + "timestamp_utc": "2026-08-01T12:00:16.314633Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T120016309010Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T120016309010Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4c2765fc7b1ad4718cc8d0b8c543284572bf135 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T120016309010Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:544f7a5ad4d401f6883d838fd43252ef7fb9d1c106b23d511d7caf3f9400b8bf +size 168218 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T120017145736Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T120017145736Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a62b687aa8567ba4d562ff0923bb9a006fd3729 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T120017145736Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:17.145328Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:16.315556Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T120017145736Z.pickle", + "timestamp_utc": "2026-08-01T12:00:17.150709Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T120017145736Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T120017145736Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6d67024af6fb1a5d0bd1028d411708ad9ede3db9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T120017145736Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de63c90163db208783c961e781681766de2eebfdeeac1c67b1c53280cec9b794 +size 168218 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T120105675551Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T120105675551Z.json new file mode 100644 index 0000000000000000000000000000000000000000..99a7a32d5adb44bb2bba4f46ce0451de6a4e609a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T120105675551Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61217477632, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61217477632, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 61217477632, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 7105637376, + "inactive_split_bytes_peak": 14813152256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137732554752, + "reserved_bytes_peak": 137732554752, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:01:05.675059Z", + "world_size": 8 + }, + "allocated_bytes_current": 61217477632, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 2, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:17.151580Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7105637376, + "inactive_split_bytes_current_delta": -1990428672, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137732554752, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137732554752, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 671, + "segment_count_current_delta": 3, + "segment_count_peak": 671, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T120105675551Z.pickle", + "timestamp_utc": "2026-08-01T12:01:05.758823Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T120105675551Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T120105675551Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b0e7f0719503beef7f67902e1287ba79d4c72ce2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T120105675551Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74b0f3be8b9a02a2a2c037da329e21705afb99f2604c15bda4d9b1283321e1a4 +size 3838316 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T120007790158Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T120007790158Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ccfa068e9759baf1664b5b8a1ef996abee607bb6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T120007790158Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 80846497280, + "active_bytes_peak_delta": 45380161024, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 80846497280, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80846497280, + "device": 3, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_peak": 8821787648, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 91515518976, + "reserved_bytes_peak": 91515518976, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:00:07.789419Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 80846497280, + "allocated_bytes_peak_delta": 45380161024, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:58:58.880269Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_current_delta": 6378380288, + "inactive_split_bytes_peak": 8821787648, + "inactive_split_bytes_peak_delta": 8661401600, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 91515518976, + "reserved_bytes_current_delta": 52271513600, + "reserved_bytes_peak": 91515518976, + "reserved_bytes_peak_delta": 52271513600, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T120007790158Z.pickle", + "timestamp_utc": "2026-08-01T12:00:07.918319Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T120007790158Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T120007790158Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fc2131eda10cdc948e2bd003ceade11548beb1f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T120007790158Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa41ccc229d5638c3577d429822b23b7ba0b9cbe8d875954c13d407a68209298 +size 5923954 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T120008937201Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T120008937201Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5dda3c67ad511239ea740a6b2adbc405c2f460a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T120008937201Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104490119680, + "active_bytes_peak_delta": 23643622400, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 3, + "inactive_split_bytes_current": 6683165696, + "inactive_split_bytes_peak": 8821787648, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:08.936683Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104490119680, + "allocated_bytes_peak_delta": 23643622400, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 80846497280, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80846497280, + "device": 3, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_peak": 8821787648, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 91515518976, + "reserved_bytes_peak": 91515518976, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:00:07.937783Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6683165696, + "inactive_split_bytes_current_delta": 171966464, + "inactive_split_bytes_peak": 8821787648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106493378560, + "reserved_bytes_current_delta": 14977859584, + "reserved_bytes_peak": 106493378560, + "reserved_bytes_peak_delta": 14977859584, + "segment_count_current": 487, + "segment_count_current_delta": 66, + "segment_count_peak": 487, + "segment_count_peak_delta": 66, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T120008937201Z.pickle", + "timestamp_utc": "2026-08-01T12:00:09.049629Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T120008937201Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T120008937201Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4274ff6aa6aca70c557378541714476ff06f6c79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T120008937201Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:864386ce20a60b907be4086f74a37bdfa5f670d3c787b21c7c17b3af78d69aa7 +size 5894150 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T120010015772Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T120010015772Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c40cae57762e768f35c4a35d6065051839728ab0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T120010015772Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104490119680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 3, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_peak": 8821787648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:10.015290Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104490119680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 3, + "inactive_split_bytes_current": 6683165696, + "inactive_split_bytes_peak": 8821787648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:09.064262Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_current_delta": 205520896, + "inactive_split_bytes_peak": 8821787648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106493378560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106493378560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T120010015772Z.pickle", + "timestamp_utc": "2026-08-01T12:00:10.128164Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T120010015772Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T120010015772Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca5319b43ccde4141a4ac04583c0571be4e17bbf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T120010015772Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b223fe3f37aac28cefd7728f4ae72a64bba503d597b121799ce98c54b7fa0802 +size 5890237 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T120012827446Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T120012827446Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ce1e43a79385d1d17ff8114cfeacc1096141ba6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T120012827446Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 17698816, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 18157308416, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:12.826842Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 17698816, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 15869440, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 3, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_peak": 8821787648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:10.142758Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 2207379456, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 5991364608, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 30802968576, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 30802968576, + "segment_count_current": 668, + "segment_count_current_delta": 181, + "segment_count_peak": 668, + "segment_count_peak_delta": 181, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T120012827446Z.pickle", + "timestamp_utc": "2026-08-01T12:00:12.952375Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T120012827446Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T120012827446Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..20955df253fa45e0f072c7224b4cc0cb3c243d06 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T120012827446Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66ca94c8e3767a0cf574e56d705cc8917d8a9b104a048981930da466df2009d3 +size 6168468 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T120013801313Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T120013801313Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b558719bfdf10bc4ed20a244c22cb6c74aa96aa0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T120013801313Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:13.800852Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:12.968894Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T120013801313Z.pickle", + "timestamp_utc": "2026-08-01T12:00:13.806751Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T120013801313Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T120013801313Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e59280e079937517b7b69fbe6f8da8108777a006 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T120013801313Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba8fb46a18f278ba1a2b197fe89a9726334d814a68da80e07cd5fa2c5da9cf42 +size 168214 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T120014637308Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T120014637308Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5dc41988a41eac03b8e752260ba5c1a034fcd1c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T120014637308Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:14.636896Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:13.807665Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T120014637308Z.pickle", + "timestamp_utc": "2026-08-01T12:00:14.642060Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T120014637308Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T120014637308Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..66fe703d703b2c802ec291e033c7c0721e7c6e3d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T120014637308Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db00e970869cdb59d105bc599882721aa590a4ebd603e30fd19a551d77ef5985 +size 168214 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T120015472774Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T120015472774Z.json new file mode 100644 index 0000000000000000000000000000000000000000..001d26738cdf8d0b514e1d59c0e33d8100c348de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T120015472774Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:15.472386Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:14.642954Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T120015472774Z.pickle", + "timestamp_utc": "2026-08-01T12:00:15.477106Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T120015472774Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T120015472774Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..07d00da34c439edfa90157e4dfac51eb80036cc9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T120015472774Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:089ecb558347c813695f2e3ec63d75aebf84307d4a22ce4fc79c3c8999690380 +size 168214 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T120016309008Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T120016309008Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a8c2155ae86db44833bd3fac8e515986b8444bcb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T120016309008Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:16.308542Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:15.477981Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T120016309008Z.pickle", + "timestamp_utc": "2026-08-01T12:00:16.314179Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T120016309008Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T120016309008Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..668228fcea29f097a819afbb09cf66549cc2d73e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T120016309008Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4255306d9302ed58ba1cec8cf73fc3313fab232fc556850f48755cfdb85865a4 +size 168214 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T120017145746Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T120017145746Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9435511b9c0805120057fb592d1eb90b99a6c480 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T120017145746Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:17.145352Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:16.315064Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T120017145746Z.pickle", + "timestamp_utc": "2026-08-01T12:00:17.150612Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T120017145746Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T120017145746Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..67e327a67cd36c65fb543e2240b8c4417585dd3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T120017145746Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6eedb4e9603a4428b8201e89da72a23f96e84d329517a1fb660b7676c47e13a +size 168214 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T120105675564Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T120105675564Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f85c2dcf878d1eab5b0e74593973a0444dd60682 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T120105675564Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61217477632, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61217477632, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 61217477632, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 7105637376, + "inactive_split_bytes_peak": 14813152256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137732554752, + "reserved_bytes_peak": 137732554752, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:01:05.675088Z", + "world_size": 8 + }, + "allocated_bytes_current": 61217477632, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 3, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:17.151497Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7105637376, + "inactive_split_bytes_current_delta": -1990428672, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137732554752, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137732554752, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 671, + "segment_count_current_delta": 3, + "segment_count_peak": 671, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T120105675564Z.pickle", + "timestamp_utc": "2026-08-01T12:01:05.761211Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T120105675564Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T120105675564Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..85f12182d66cb36c61ad1ceb8a14f23c6b3d7927 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T120105675564Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04ca2464e111cf2fad45489400bd8b4664cfab3f55351d24ac68625ad85e5066 +size 3838299 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T120007789764Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T120007789764Z.json new file mode 100644 index 0000000000000000000000000000000000000000..247419555a3c46bfb230d3f83a9e4f65244e714e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T120007789764Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72458362880, + "active_bytes_current_delta": 37226465280, + "active_bytes_peak": 80847479296, + "active_bytes_peak_delta": 45380700672, + "after": { + "active_bytes_current": 72458362880, + "active_bytes_peak": 80847479296, + "allocated_bytes_current": 72458362880, + "allocated_bytes_peak": 80847479296, + "device": 4, + "inactive_split_bytes_current": 6069493760, + "inactive_split_bytes_peak": 8280257024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 91674902528, + "reserved_bytes_peak": 91674902528, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:00:07.789183Z", + "world_size": 8 + }, + "allocated_bytes_current": 72458362880, + "allocated_bytes_current_delta": 37226465280, + "allocated_bytes_peak": 80847479296, + "allocated_bytes_peak_delta": 45380700672, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:58:58.879603Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6069493760, + "inactive_split_bytes_current_delta": 5937117184, + "inactive_split_bytes_peak": 8280257024, + "inactive_split_bytes_peak_delta": 8126847488, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 91674902528, + "reserved_bytes_current_delta": 52430897152, + "reserved_bytes_peak": 91674902528, + "reserved_bytes_peak_delta": 52430897152, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T120007789764Z.pickle", + "timestamp_utc": "2026-08-01T12:00:07.910211Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T120007789764Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T120007789764Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b708928c11635c4db8f9e2639595b0e6242afeb7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T120007789764Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37f9c37cddbf0f070ff4946ca3ec7180773bae1242098895df3f8fa0753a24bc +size 5932698 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T120008936884Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T120008936884Z.json new file mode 100644 index 0000000000000000000000000000000000000000..796cfd9fbadbe81961588b0f1462b2f0734f252d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T120008936884Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72458362880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104492543488, + "active_bytes_peak_delta": 23645064192, + "after": { + "active_bytes_current": 72458362880, + "active_bytes_peak": 104492543488, + "allocated_bytes_current": 72458362880, + "allocated_bytes_peak": 104492543488, + "device": 4, + "inactive_split_bytes_current": 6772039680, + "inactive_split_bytes_peak": 8392815104, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106329800704, + "reserved_bytes_peak": 106329800704, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T12:00:08.936483Z", + "world_size": 8 + }, + "allocated_bytes_current": 72458362880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104492543488, + "allocated_bytes_peak_delta": 23645064192, + "before": { + "active_bytes_current": 72458362880, + "active_bytes_peak": 80847479296, + "allocated_bytes_current": 72458362880, + "allocated_bytes_peak": 80847479296, + "device": 4, + "inactive_split_bytes_current": 6069493760, + "inactive_split_bytes_peak": 8280257024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 91674902528, + "reserved_bytes_peak": 91674902528, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:00:07.927126Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6772039680, + "inactive_split_bytes_current_delta": 702545920, + "inactive_split_bytes_peak": 8392815104, + "inactive_split_bytes_peak_delta": 112558080, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106329800704, + "reserved_bytes_current_delta": 14654898176, + "reserved_bytes_peak": 106329800704, + "reserved_bytes_peak_delta": 14654898176, + "segment_count_current": 485, + "segment_count_current_delta": 64, + "segment_count_peak": 485, + "segment_count_peak_delta": 64, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T120008936884Z.pickle", + "timestamp_utc": "2026-08-01T12:00:09.047759Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T120008936884Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T120008936884Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..48f6c6241af242d29464fbb15840cca8068bafcd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T120008936884Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c8675c48da040ccba83e6024bc99dff25b5869d6c923e79ed35aef8533a4330 +size 5946004 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T120010015503Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T120010015503Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c6b1d05d65ed1a8383df018270b9398a98400bdc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T120010015503Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72458362880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104492543488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72458362880, + "active_bytes_peak": 104492543488, + "allocated_bytes_current": 72458362880, + "allocated_bytes_peak": 104492543488, + "device": 4, + "inactive_split_bytes_current": 7185178624, + "inactive_split_bytes_peak": 8392815104, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106329800704, + "reserved_bytes_peak": 106329800704, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T12:00:10.015106Z", + "world_size": 8 + }, + "allocated_bytes_current": 72458362880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104492543488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72458362880, + "active_bytes_peak": 104492543488, + "allocated_bytes_current": 72458362880, + "allocated_bytes_peak": 104492543488, + "device": 4, + "inactive_split_bytes_current": 6772039680, + "inactive_split_bytes_peak": 8392815104, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106329800704, + "reserved_bytes_peak": 106329800704, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T12:00:09.061639Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7185178624, + "inactive_split_bytes_current_delta": 413138944, + "inactive_split_bytes_peak": 8392815104, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106329800704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106329800704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T120010015503Z.pickle", + "timestamp_utc": "2026-08-01T12:00:10.125136Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T120010015503Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T120010015503Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..740d20e866dfd7b631cfebdf6e556fe3778fc110 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T120010015503Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:922524d419e89ad8f125126497e7e7112c910d74844e36dcf870fadf65cef968 +size 6021751 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T120012827235Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T120012827235Z.json new file mode 100644 index 0000000000000000000000000000000000000000..81975762193a488dfb2ccabc261e32de2f5f7281 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T120012827235Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475406336, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 122654695424, + "active_bytes_peak_delta": 18162151936, + "after": { + "active_bytes_current": 72475406336, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 72475406336, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_peak": 15521341440, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137610919936, + "reserved_bytes_peak": 137610919936, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T12:00:12.826708Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475406336, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104502639616, + "allocated_bytes_peak_delta": 10096128, + "before": { + "active_bytes_current": 72458362880, + "active_bytes_peak": 104492543488, + "allocated_bytes_current": 72458362880, + "allocated_bytes_peak": 104492543488, + "device": 4, + "inactive_split_bytes_current": 7185178624, + "inactive_split_bytes_peak": 8392815104, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106329800704, + "reserved_bytes_peak": 106329800704, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T12:00:10.138600Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_current_delta": 2298212352, + "inactive_split_bytes_peak": 15521341440, + "inactive_split_bytes_peak_delta": 7128526336, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137610919936, + "reserved_bytes_current_delta": 31281119232, + "reserved_bytes_peak": 137610919936, + "reserved_bytes_peak_delta": 31281119232, + "segment_count_current": 665, + "segment_count_current_delta": 180, + "segment_count_peak": 665, + "segment_count_peak_delta": 180, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T120012827235Z.pickle", + "timestamp_utc": "2026-08-01T12:00:12.953459Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T120012827235Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T120012827235Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7263799d567b8a32e12b6ad7316c435cb8e3438e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T120012827235Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ed11e85af33800a07e2a409487fde39fc5b8b045fc01c79f1b8132a96c71b8c +size 6326796 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T120013801179Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T120013801179Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ba7e5d337b985cfb4d9d97525f48865d8127979a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T120013801179Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475406336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122654695424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475406336, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 72475406336, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_peak": 15521341440, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137610919936, + "reserved_bytes_peak": 137610919936, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T12:00:13.800702Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475406336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104502639616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475406336, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 72475406336, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_peak": 15521341440, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137610919936, + "reserved_bytes_peak": 137610919936, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T12:00:12.968864Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15521341440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137610919936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137610919936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 665, + "segment_count_current_delta": 0, + "segment_count_peak": 665, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T120013801179Z.pickle", + "timestamp_utc": "2026-08-01T12:00:13.806123Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T120013801179Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T120013801179Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1362d9e078cb9d71f1be3e62477d4068c16c74e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T120013801179Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d5eddf616339fdf95adeda1d459ac1396dc1a1d72ede19fc1d815b27d663a11 +size 172274 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T120014637286Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T120014637286Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0c7548d19dc14b0d2a370f0ee94df3310d27ac39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T120014637286Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475406336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122654695424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475406336, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 72475406336, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_peak": 15521341440, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137610919936, + "reserved_bytes_peak": 137610919936, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T12:00:14.636803Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475406336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104502639616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475406336, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 72475406336, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_peak": 15521341440, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137610919936, + "reserved_bytes_peak": 137610919936, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T12:00:13.807039Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15521341440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137610919936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137610919936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 665, + "segment_count_current_delta": 0, + "segment_count_peak": 665, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T120014637286Z.pickle", + "timestamp_utc": "2026-08-01T12:00:14.642117Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T120014637286Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T120014637286Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b239edf240c84441d47286c04d347273ce97a98 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T120014637286Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01c03b2e7d455dd5580f036e81de4697ec3ee83b22b10466644082b1a0f81a4e +size 172243 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T120015472699Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T120015472699Z.json new file mode 100644 index 0000000000000000000000000000000000000000..233f2803983d7a6be370eed34e7b7cdef2a1343d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T120015472699Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475406336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122654695424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475406336, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 72475406336, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_peak": 15521341440, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137610919936, + "reserved_bytes_peak": 137610919936, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T12:00:15.472310Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475406336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104502639616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475406336, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 72475406336, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_peak": 15521341440, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137610919936, + "reserved_bytes_peak": 137610919936, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T12:00:14.643011Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15521341440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137610919936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137610919936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 665, + "segment_count_current_delta": 0, + "segment_count_peak": 665, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T120015472699Z.pickle", + "timestamp_utc": "2026-08-01T12:00:15.477237Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T120015472699Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T120015472699Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a9a54d5acfc70f3ab97e500ef393e0d473e1688 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T120015472699Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9056715eff31cbd8d259fb5f7665987bfa8a5fb97d44ebd4c6bbb310541ba9c7 +size 172274 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T120016308968Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T120016308968Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3c57bfe83f065ae78f35797573dca6311bc275e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T120016308968Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475406336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122654695424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475406336, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 72475406336, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_peak": 15521341440, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137610919936, + "reserved_bytes_peak": 137610919936, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T12:00:16.308477Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475406336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104502639616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475406336, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 72475406336, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_peak": 15521341440, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137610919936, + "reserved_bytes_peak": 137610919936, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T12:00:15.478112Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15521341440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137610919936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137610919936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 665, + "segment_count_current_delta": 0, + "segment_count_peak": 665, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T120016308968Z.pickle", + "timestamp_utc": "2026-08-01T12:00:16.314497Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T120016308968Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T120016308968Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8f59bafea649c5bd85e5946ce4e88a999c374fa4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T120016308968Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0fdacf8ee43995d364fb044098c88facc0ba0cf8becc1be986360ea21def914 +size 172243 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T120017145686Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T120017145686Z.json new file mode 100644 index 0000000000000000000000000000000000000000..00dfa9601a5615afbfe7255ca653a9e6bef77f30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T120017145686Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475406336, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122654695424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475406336, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 72475406336, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_peak": 15521341440, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137610919936, + "reserved_bytes_peak": 137610919936, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T12:00:17.145268Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475406336, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104502639616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475406336, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 72475406336, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_peak": 15521341440, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137610919936, + "reserved_bytes_peak": 137610919936, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T12:00:16.315413Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15521341440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137610919936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137610919936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 665, + "segment_count_current_delta": 0, + "segment_count_peak": 665, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T120017145686Z.pickle", + "timestamp_utc": "2026-08-01T12:00:17.150773Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T120017145686Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T120017145686Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d6a15b26e7d4d01023a46aa808d11d843e54d97 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T120017145686Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58eeb064dd88082978ba27b3468172c3622a9f0f86830b4684cbca7f93fcc87a +size 172274 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T120105675540Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T120105675540Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4143918f376c253380d466a941a218f710b93f74 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T120105675540Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61218125824, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 122654695424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61218125824, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 61218125824, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 7490865152, + "inactive_split_bytes_peak": 15521341440, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137633988608, + "reserved_bytes_peak": 137633988608, + "segment_count_current": 667, + "segment_count_peak": 667, + "timestamp_utc": "2026-08-01T12:01:05.675049Z", + "world_size": 8 + }, + "allocated_bytes_current": 61218125824, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104502639616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475406336, + "active_bytes_peak": 122654695424, + "allocated_bytes_current": 72475406336, + "allocated_bytes_peak": 104502639616, + "device": 4, + "inactive_split_bytes_current": 9483390976, + "inactive_split_bytes_peak": 15521341440, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137610919936, + "reserved_bytes_peak": 137610919936, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T12:00:17.151680Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7490865152, + "inactive_split_bytes_current_delta": -1992525824, + "inactive_split_bytes_peak": 15521341440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137633988608, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 137633988608, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 667, + "segment_count_current_delta": 2, + "segment_count_peak": 667, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T120105675540Z.pickle", + "timestamp_utc": "2026-08-01T12:01:05.761477Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T120105675540Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T120105675540Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3714163672281bcc081b39f8dc03de009ab0eeab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T120105675540Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c80a45ef0d199dbd63913487157b21ede18bc1c3a6ec199433ffde0685bcb18d +size 4092081 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T120007789621Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T120007789621Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2b5dc05d09fad9fcdb67d1c23b37feecd413d12b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T120007789621Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 80846497280, + "active_bytes_peak_delta": 45380161024, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 80846497280, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80846497280, + "device": 5, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_peak": 8821705728, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 91513421824, + "reserved_bytes_peak": 91513421824, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T12:00:07.789048Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 80846497280, + "allocated_bytes_peak_delta": 45380161024, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:58:58.880181Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_current_delta": 6378380288, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 8661319680, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 91513421824, + "reserved_bytes_current_delta": 52269416448, + "reserved_bytes_peak": 91513421824, + "reserved_bytes_peak_delta": 52269416448, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T120007789621Z.pickle", + "timestamp_utc": "2026-08-01T12:00:07.908931Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T120007789621Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T120007789621Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d7f603e0fe079fbd12aed335988507c00274cca5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T120007789621Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3851451994da63d3488ecf26915c6574437e75f0850839dfbab5057b25b201c1 +size 5923792 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T120008936900Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T120008936900Z.json new file mode 100644 index 0000000000000000000000000000000000000000..955e48732b2d1a0d83ffd0564cb031f5518de5c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T120008936900Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104490119680, + "active_bytes_peak_delta": 23643622400, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 5, + "inactive_split_bytes_current": 6685262848, + "inactive_split_bytes_peak": 8821705728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:08.936496Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104490119680, + "allocated_bytes_peak_delta": 23643622400, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 80846497280, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80846497280, + "device": 5, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_peak": 8821705728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 91513421824, + "reserved_bytes_peak": 91513421824, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T12:00:07.928291Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6685262848, + "inactive_split_bytes_current_delta": 174063616, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106493378560, + "reserved_bytes_current_delta": 14979956736, + "reserved_bytes_peak": 106493378560, + "reserved_bytes_peak_delta": 14979956736, + "segment_count_current": 487, + "segment_count_current_delta": 67, + "segment_count_peak": 487, + "segment_count_peak_delta": 67, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T120008936900Z.pickle", + "timestamp_utc": "2026-08-01T12:00:09.048365Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T120008936900Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T120008936900Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e3383447227a7a454deb2962d18d27c28a982100 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T120008936900Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4b5eae23cc826c890a489ac4c0bc1f46e799f8f9ea3d8ba3dacaead130a283e +size 5894440 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T120010015515Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T120010015515Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b047224fdc8628f5dc671160b14c0f1bcad6e502 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T120010015515Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104490119680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 5, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_peak": 8821705728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:10.015123Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104490119680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 5, + "inactive_split_bytes_current": 6685262848, + "inactive_split_bytes_peak": 8821705728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:09.064069Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_current_delta": 203423744, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106493378560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106493378560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T120010015515Z.pickle", + "timestamp_utc": "2026-08-01T12:00:10.125699Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T120010015515Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T120010015515Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e0754c993b65167148f0f55e37f57f0028be29c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T120010015515Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc644d27d5df0b98aec6eec2e47c14b6faa0de44c7db2b65ed4fe9dc2a37d783 +size 5890281 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T120012827391Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T120012827391Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8bae6b52ee4cd4691b09900474d592bdfd7b761b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T120012827391Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 17698816, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 18157308416, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:12.826763Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 17698816, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 15869440, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 5, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_peak": 8821705728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:10.141046Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 2207379456, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 5991446528, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 30802968576, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 30802968576, + "segment_count_current": 668, + "segment_count_current_delta": 181, + "segment_count_peak": 668, + "segment_count_peak_delta": 181, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T120012827391Z.pickle", + "timestamp_utc": "2026-08-01T12:00:12.951775Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T120012827391Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T120012827391Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f871f3577d4dbf6c863d06d2c09b92343a5f4d60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T120012827391Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58eb57a9555e037a39d09c473594a99a10b47af32039ca2f73bd6346afe114fd +size 6168477 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T120013801184Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T120013801184Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65943beadc40a55b992ff9084976f4eb64197ccd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T120013801184Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:13.800704Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:12.969594Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T120013801184Z.pickle", + "timestamp_utc": "2026-08-01T12:00:13.806204Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T120013801184Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T120013801184Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0119139f88711aceb292eec12d26c4575d39a86c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T120013801184Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c4c0758cc9f85bf6518e4de01e77861ce20dd92d8626426fe68ad32298faf3f +size 168223 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T120014637204Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T120014637204Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bc1b1b9656c64f88e4dbc46124daac295a52e44d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T120014637204Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:14.636793Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:13.807073Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T120014637204Z.pickle", + "timestamp_utc": "2026-08-01T12:00:14.641904Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T120014637204Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T120014637204Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..39d4b7b24c49c23692d77553cded22e2a2a7b25b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T120014637204Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d2eca733c075d23033ecfed8b8efcf5079e4862930d8b92fdd211b62da4597d +size 168223 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T120015472778Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T120015472778Z.json new file mode 100644 index 0000000000000000000000000000000000000000..92307b6d6521fc2699e3d75ef83110780056cd9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T120015472778Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:15.472338Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:14.642775Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T120015472778Z.pickle", + "timestamp_utc": "2026-08-01T12:00:15.477315Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T120015472778Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T120015472778Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ad1f2a3d3a5cb42b00a803db0dfc319c6ed4246 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T120015472778Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2964da0b81a8c1e9c2be551dc49ce0cdc07fbf62ddaeb26cf5795438d42fce00 +size 168223 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T120016308974Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T120016308974Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ca8e4a322594c1441a5195d39cc54b3ab44dd87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T120016308974Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:16.308487Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:15.478175Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T120016308974Z.pickle", + "timestamp_utc": "2026-08-01T12:00:16.314579Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T120016308974Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T120016308974Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2450835f87b581602dcabdef0c07c55bba46c3d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T120016308974Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0316f65e65edd0557264d53f6652f3e042fb1e819172cf436650592eaa9b8687 +size 168223 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T120017145691Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T120017145691Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a8618127d0c811732fe134cc3286c454e48ccd41 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T120017145691Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:17.145269Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:16.315483Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T120017145691Z.pickle", + "timestamp_utc": "2026-08-01T12:00:17.150415Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T120017145691Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T120017145691Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fa3608185c3a46cfc90741044327d9f2206f9ceb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T120017145691Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4261468b0c82dda4c6159c174d6dd22d47998c69a4aefcfe2f8bda7e54383fe9 +size 168223 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T120105675567Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T120105675567Z.json new file mode 100644 index 0000000000000000000000000000000000000000..387fae0af32e120c37477053d9c7e8cba2bfae90 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T120105675567Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61217477632, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61217477632, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 61217477632, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 7105637376, + "inactive_split_bytes_peak": 14813152256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137732554752, + "reserved_bytes_peak": 137732554752, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:01:05.675073Z", + "world_size": 8 + }, + "allocated_bytes_current": 61217477632, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 5, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:17.151265Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7105637376, + "inactive_split_bytes_current_delta": -1990428672, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137732554752, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137732554752, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 671, + "segment_count_current_delta": 3, + "segment_count_peak": 671, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T120105675567Z.pickle", + "timestamp_utc": "2026-08-01T12:01:05.762318Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T120105675567Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T120105675567Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..26c8a23188781683606a19301e98cf2a542df63e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T120105675567Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bec78a1a0e6064e87e7cdb58b1d53406a98d01a1c5c779738d2f33c26f80229 +size 3838276 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T120007789686Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T120007789686Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1748627d287b159d0b846bf50fcc699b90ed83e1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T120007789686Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 80846497280, + "active_bytes_peak_delta": 45380161024, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 80846497280, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80846497280, + "device": 6, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_peak": 8821705728, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 91513421824, + "reserved_bytes_peak": 91513421824, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T12:00:07.789100Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 80846497280, + "allocated_bytes_peak_delta": 45380161024, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:58:58.880184Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_current_delta": 6378380288, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 8661319680, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 91513421824, + "reserved_bytes_current_delta": 52269416448, + "reserved_bytes_peak": 91513421824, + "reserved_bytes_peak_delta": 52269416448, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T120007789686Z.pickle", + "timestamp_utc": "2026-08-01T12:00:07.908562Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T120007789686Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T120007789686Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eaa091bba0cdf5cc4854a7625290bc2da53c857b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T120007789686Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11a84eeb8a94c41e3181a24ec2733c60a7cd644a30d2b186d5a58034135fc8fe +size 5923789 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T120008936906Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T120008936906Z.json new file mode 100644 index 0000000000000000000000000000000000000000..830efce25e78b3126d4d1731b197cbe85541a953 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T120008936906Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104490119680, + "active_bytes_peak_delta": 23643622400, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 6, + "inactive_split_bytes_current": 6685262848, + "inactive_split_bytes_peak": 8821705728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106491281408, + "reserved_bytes_peak": 106491281408, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T12:00:08.936495Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104490119680, + "allocated_bytes_peak_delta": 23643622400, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 80846497280, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80846497280, + "device": 6, + "inactive_split_bytes_current": 6511199232, + "inactive_split_bytes_peak": 8821705728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 91513421824, + "reserved_bytes_peak": 91513421824, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T12:00:07.927833Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6685262848, + "inactive_split_bytes_current_delta": 174063616, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106491281408, + "reserved_bytes_current_delta": 14977859584, + "reserved_bytes_peak": 106491281408, + "reserved_bytes_peak_delta": 14977859584, + "segment_count_current": 486, + "segment_count_current_delta": 66, + "segment_count_peak": 486, + "segment_count_peak_delta": 66, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T120008936906Z.pickle", + "timestamp_utc": "2026-08-01T12:00:09.048820Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T120008936906Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T120008936906Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ef8cb31d3303d6b8f5bcdf4b2a3cbefb7e39caf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T120008936906Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:269c05d32b35ba916273ccc7e9b90d5613be9fb8271cf6e9d1624523dab56487 +size 5894282 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T120010015502Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T120010015502Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9614946ecfa4d36f9a4dd5198c9d7c53e74d2c81 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T120010015502Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104490119680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 6, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_peak": 8821705728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106491281408, + "reserved_bytes_peak": 106491281408, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T12:00:10.015102Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104490119680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 6, + "inactive_split_bytes_current": 6685262848, + "inactive_split_bytes_peak": 8821705728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106491281408, + "reserved_bytes_peak": 106491281408, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T12:00:09.064573Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_current_delta": 203423744, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106491281408, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106491281408, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T120010015502Z.pickle", + "timestamp_utc": "2026-08-01T12:00:10.125238Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T120010015502Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T120010015502Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ef047bd5e37900056b9d4f0e516841e1a99e10ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T120010015502Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a7dd4e743e9bb2b7166c03d7b4277f639b4092959a9ea2174fb978c2ea65c78 +size 5890137 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T120012827401Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T120012827401Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a1a8cd2f8386ca5f89349fd830b81e4680e94628 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T120012827401Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 17698816, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 18157308416, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:12.826829Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 17698816, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 15869440, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104490119680, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104490119680, + "device": 6, + "inactive_split_bytes_current": 6888686592, + "inactive_split_bytes_peak": 8821705728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106491281408, + "reserved_bytes_peak": 106491281408, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T12:00:10.140822Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 2207379456, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 5991446528, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 30805065728, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 30805065728, + "segment_count_current": 668, + "segment_count_current_delta": 182, + "segment_count_peak": 668, + "segment_count_peak_delta": 182, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T120012827401Z.pickle", + "timestamp_utc": "2026-08-01T12:00:12.952597Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T120012827401Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T120012827401Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..654763a09abe5a51925923b48735057ad06db3eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T120012827401Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12928303cdf18e953fb439346938017c5de5b214aa408848bd4a5d14d77d4404 +size 6168414 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T120013801157Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T120013801157Z.json new file mode 100644 index 0000000000000000000000000000000000000000..366188c7b3f161c4d75ac246500e026c8b4e7c19 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T120013801157Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:13.800687Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:12.969902Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T120013801157Z.pickle", + "timestamp_utc": "2026-08-01T12:00:13.806263Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T120013801157Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T120013801157Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..645c8a4ad1676d9451c68be006c93a784b9f4c05 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T120013801157Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b83a16dd568d1a398b2dd143dc105d9f5fe4c0c13776da9fbafad4546734b58e +size 168212 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T120014637204Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T120014637204Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1fe757cde5194860694c525e79218a4550753b57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T120014637204Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:14.636789Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:13.807135Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T120014637204Z.pickle", + "timestamp_utc": "2026-08-01T12:00:14.642021Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T120014637204Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T120014637204Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..22249e6bf607ffc93048bcef00e1409d10ed60f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T120014637204Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3e1488ff6604c5895cdae89b0c9dbf93ce458d07dc3bdf266467de3107ab63b +size 168212 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T120015472747Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T120015472747Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4cc514ab028ede1746150e8faf4d96003ae06e0f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T120015472747Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:15.472318Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:14.642895Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T120015472747Z.pickle", + "timestamp_utc": "2026-08-01T12:00:15.477273Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T120015472747Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T120015472747Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..33abefcf4a5d30f0e28d2df5cafbe6c6d5be54fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T120015472747Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0584af4a34b3adef2bb69c64fa9718deb7f3a9c12bc5ed3910f1b64559fdb812 +size 168212 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T120016309017Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T120016309017Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc3dc7314113cbc6ca0e9e394a68fb19fd3cf932 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T120016309017Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:16.308512Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:15.478126Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T120016309017Z.pickle", + "timestamp_utc": "2026-08-01T12:00:16.314537Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T120016309017Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T120016309017Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bcf2cbaf0896af55c6cf6690cc76e0cc36947168 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T120016309017Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e5bf2c35311be44e019fae9d3659472d05b645ab91c9e9ac17766d4e62eff25 +size 168212 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T120017145669Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T120017145669Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bee34c328a524df6e46855287a633ecf41335f02 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T120017145669Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758144, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:17.145258Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758144, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:16.315446Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T120017145669Z.pickle", + "timestamp_utc": "2026-08-01T12:00:17.150369Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T120017145669Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T120017145669Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f29cd2a0cd8312230a3434aeafafa651221fe3c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T120017145669Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99b27082963329427b746742e2ff14461bf56d502841315ae22ae1f85bab1d99 +size 168212 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T120105675525Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T120105675525Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b6da446484a888c7c60e04fbff880deddfa1f8a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T120105675525Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61217477632, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 122647428096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61217477632, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 61217477632, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 7105637376, + "inactive_split_bytes_peak": 14813152256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137732554752, + "reserved_bytes_peak": 137732554752, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:01:05.675050Z", + "world_size": 8 + }, + "allocated_bytes_current": 61217477632, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104505989120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758144, + "active_bytes_peak": 122647428096, + "allocated_bytes_current": 72474758144, + "allocated_bytes_peak": 104505989120, + "device": 6, + "inactive_split_bytes_current": 9096066048, + "inactive_split_bytes_peak": 14813152256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:17.151230Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7105637376, + "inactive_split_bytes_current_delta": -1990428672, + "inactive_split_bytes_peak": 14813152256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137732554752, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137732554752, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 671, + "segment_count_current_delta": 3, + "segment_count_peak": 671, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T120105675525Z.pickle", + "timestamp_utc": "2026-08-01T12:01:05.755911Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T120105675525Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T120105675525Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ddf08d2a476cc9a722fc5cdd5118a5d69e239f39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T120105675525Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b433fde2b6069382e5b55287957cfc34b2e6add86fc9b7dab404c84c2fde0f87 +size 3838272 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T120007789589Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T120007789589Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a6666b3b2ad5f4ecec6ad2cfe39b6a4e1d7d8082 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T120007789589Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059840, + "active_bytes_current_delta": 37225604608, + "active_bytes_peak": 80846497280, + "active_bytes_peak_delta": 45380161024, + "after": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 80846497280, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 80846497280, + "device": 7, + "inactive_split_bytes_current": 6511198720, + "inactive_split_bytes_peak": 8821705728, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 91513421824, + "reserved_bytes_peak": 91513421824, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T12:00:07.789019Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059840, + "allocated_bytes_current_delta": 37225604608, + "allocated_bytes_peak": 80846497280, + "allocated_bytes_peak_delta": 45380161024, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T11:58:58.880117Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6511198720, + "inactive_split_bytes_current_delta": 6378379776, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 8661319680, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 91513421824, + "reserved_bytes_current_delta": 52269416448, + "reserved_bytes_peak": 91513421824, + "reserved_bytes_peak_delta": 52269416448, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T120007789589Z.pickle", + "timestamp_utc": "2026-08-01T12:00:07.905199Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T120007789589Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T120007789589Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a76433aa4d4674d8401fc3f7a38af9b2833826e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T120007789589Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6262dee6c23d6be70ce6617c62ab787584776d292daf78ac19051977ff5e3ab3 +size 5925383 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T120008936964Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T120008936964Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a642eab40b374db5c00dc8ee7fb3514215db6f5e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T120008936964Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104490120192, + "active_bytes_peak_delta": 23643622912, + "after": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 104490120192, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 104490120192, + "device": 7, + "inactive_split_bytes_current": 6683165184, + "inactive_split_bytes_peak": 8821705728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:08.936548Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104490120192, + "allocated_bytes_peak_delta": 23643622912, + "before": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 80846497280, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 80846497280, + "device": 7, + "inactive_split_bytes_current": 6511198720, + "inactive_split_bytes_peak": 8821705728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 91513421824, + "reserved_bytes_peak": 91513421824, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T12:00:07.922195Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6683165184, + "inactive_split_bytes_current_delta": 171966464, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106493378560, + "reserved_bytes_current_delta": 14979956736, + "reserved_bytes_peak": 106493378560, + "reserved_bytes_peak_delta": 14979956736, + "segment_count_current": 487, + "segment_count_current_delta": 67, + "segment_count_peak": 487, + "segment_count_peak_delta": 67, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T120008936964Z.pickle", + "timestamp_utc": "2026-08-01T12:00:09.047100Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T120008936964Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T120008936964Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..74ea63281215fa9aaa2bc265b8fe2cc2e2cc42a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T120008936964Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93173d2d07489aa57aba934bd3cfd1df1b1db57488eebd0cff042f748956836f +size 5896149 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T120010015516Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T120010015516Z.json new file mode 100644 index 0000000000000000000000000000000000000000..43a162ea485d9791680796c092acac33dd1d6353 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T120010015516Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104490120192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 104490120192, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 104490120192, + "device": 7, + "inactive_split_bytes_current": 6888686080, + "inactive_split_bytes_peak": 8821705728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:10.015112Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104490120192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 104490120192, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 104490120192, + "device": 7, + "inactive_split_bytes_current": 6683165184, + "inactive_split_bytes_peak": 8821705728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:09.060935Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6888686080, + "inactive_split_bytes_current_delta": 205520896, + "inactive_split_bytes_peak": 8821705728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106493378560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106493378560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T120010015516Z.pickle", + "timestamp_utc": "2026-08-01T12:00:10.120913Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T120010015516Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T120010015516Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..41edc45e1bc5c6b08f724cada16ac82a2d158a57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T120010015516Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d51e8fa8046781405b44fa8f3e6a32090de47e7e33723150fff6bd8fbf143db1 +size 5892029 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T120012827302Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T120012827302Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf3653d8514e2d0662af13b856e0b91e4e9df19b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T120012827302Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758656, + "active_bytes_current_delta": 17698816, + "active_bytes_peak": 122647428608, + "active_bytes_peak_delta": 18157308416, + "after": { + "active_bytes_current": 72474758656, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 72474758656, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_peak": 14813151744, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:12.826748Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758656, + "allocated_bytes_current_delta": 17698816, + "allocated_bytes_peak": 104505989632, + "allocated_bytes_peak_delta": 15869440, + "before": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 104490120192, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 104490120192, + "device": 7, + "inactive_split_bytes_current": 6888686080, + "inactive_split_bytes_peak": 8821705728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106493378560, + "reserved_bytes_peak": 106493378560, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:00:10.134489Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_current_delta": 2207379456, + "inactive_split_bytes_peak": 14813151744, + "inactive_split_bytes_peak_delta": 5991446016, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 30802968576, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 30802968576, + "segment_count_current": 668, + "segment_count_current_delta": 181, + "segment_count_peak": 668, + "segment_count_peak_delta": 181, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T120012827302Z.pickle", + "timestamp_utc": "2026-08-01T12:00:12.948072Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T120012827302Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T120012827302Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c8d9a100f61e7ac4d364dcd2ca51217ecdfa3b99 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T120012827302Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d87a1513beded1ac1f853abde3ed0ad791d8aa4c2925b4b0360a77c1cfb6066a +size 6170203 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T120013801200Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T120013801200Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aa77521d427b1d777389e3fbc90af702a7d554e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T120013801200Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758656, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 72474758656, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_peak": 14813151744, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:13.800733Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758656, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 72474758656, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_peak": 14813151744, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:12.963367Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813151744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T120013801200Z.pickle", + "timestamp_utc": "2026-08-01T12:00:13.805992Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T120013801200Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T120013801200Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d036c7c5d72601e8364faa2a65549eae72aceb11 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T120013801200Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a06e029929f19f10abcdc3ccfe2d7e17b7dbbb6a51689d64556751eb0d75ae4 +size 169978 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T120014637255Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T120014637255Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5b6f0d861834d24a405ae44b08f641cb063db9ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T120014637255Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758656, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 72474758656, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_peak": 14813151744, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:14.636841Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758656, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 72474758656, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_peak": 14813151744, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:13.806922Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813151744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T120014637255Z.pickle", + "timestamp_utc": "2026-08-01T12:00:14.641842Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T120014637255Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T120014637255Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..acfce6caf1e0b737a75cafe9898d8d70417e53fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T120014637255Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51747d7d81d5f63b37ad0d85265625c6c059686253fa0dc3b5a5a858ea2e614c +size 169978 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T120015472760Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T120015472760Z.json new file mode 100644 index 0000000000000000000000000000000000000000..919d7a047a7c47c5f1c807d9b2c1ee2d01225df3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T120015472760Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758656, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 72474758656, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_peak": 14813151744, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:15.472359Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758656, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 72474758656, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_peak": 14813151744, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:14.642727Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813151744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T120015472760Z.pickle", + "timestamp_utc": "2026-08-01T12:00:15.477143Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T120015472760Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T120015472760Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..53d7c380ed7732e57aae75e8b8e456231ef4fd70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T120015472760Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be9b3deb65c2ba8f33273f65abb0b732fb2d25b5c0bf8917784b5a85fb23aeb9 +size 169978 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T120016309019Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T120016309019Z.json new file mode 100644 index 0000000000000000000000000000000000000000..77e73d9734c8a6dfc04cc7eff3b98f2ce1ee6d84 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T120016309019Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758656, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 72474758656, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_peak": 14813151744, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:16.308518Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758656, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 72474758656, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_peak": 14813151744, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:15.477983Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813151744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T120016309019Z.pickle", + "timestamp_utc": "2026-08-01T12:00:16.314331Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T120016309019Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T120016309019Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e01739f4151861628ae74e0b8435173c9e4e825a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T120016309019Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36890c1ce6a2805e91a404ac264deec4759d678a44681dc9e384443cc7fd977b +size 169978 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T120017145720Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T120017145720Z.json new file mode 100644 index 0000000000000000000000000000000000000000..73256832bf290f6f0b818b26283a5f4bab27bcc5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T120017145720Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474758656, + "active_bytes_current_delta": 0, + "active_bytes_peak": 122647428608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474758656, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 72474758656, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_peak": 14813151744, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:17.145307Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474758656, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104505989632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758656, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 72474758656, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_peak": 14813151744, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:16.315212Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14813151744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 668, + "segment_count_current_delta": 0, + "segment_count_peak": 668, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T120017145720Z.pickle", + "timestamp_utc": "2026-08-01T12:00:17.150285Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T120017145720Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T120017145720Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dfa01c1fafc3ec39d880730151ecb95eb222d6dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T120017145720Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aef8baed2b6e6c291fde1c0072e1398c7f68b3b50c78679551586a61af70a328 +size 169978 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T120105675525Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T120105675525Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87c2c9bc407eda22ea6d1826f180fee0cf4e15cf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T120105675525Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61217478144, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 122647428608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61217478144, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 61217478144, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 7105636864, + "inactive_split_bytes_peak": 14813151744, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137732554752, + "reserved_bytes_peak": 137732554752, + "segment_count_current": 671, + "segment_count_peak": 671, + "timestamp_utc": "2026-08-01T12:01:05.675052Z", + "world_size": 8 + }, + "allocated_bytes_current": 61217478144, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104505989632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474758656, + "active_bytes_peak": 122647428608, + "allocated_bytes_current": 72474758656, + "allocated_bytes_peak": 104505989632, + "device": 7, + "inactive_split_bytes_current": 9096065536, + "inactive_split_bytes_peak": 14813151744, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T12:00:17.151138Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7105636864, + "inactive_split_bytes_current_delta": -1990428672, + "inactive_split_bytes_peak": 14813151744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137732554752, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137732554752, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 671, + "segment_count_current_delta": 3, + "segment_count_peak": 671, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T120105675525Z.pickle", + "timestamp_utc": "2026-08-01T12:01:05.755449Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T120105675525Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T120105675525Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03df81761fa559e3a04e2b21e9a1be8683d6b40c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T120105675525Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbb0b80d27845949de4fc5cc5d2a7ff4b7a3010847f0917863097b7dfbf6725c +size 3840034 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..834b74dc02a2dee422b9c195cf8411209990eb58 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 2048 +capacity: 1.25 +expert_dispatch_overlap: on +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6006 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 225 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T120404665310Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T120404665310Z.json new file mode 100644 index 0000000000000000000000000000000000000000..378e326f9cde7f626c23fc57220a8706c24808d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T120404665310Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872599552, + "active_bytes_current_delta": 31641144320, + "active_bytes_peak": 80934678528, + "active_bytes_peak_delta": 45468342272, + "after": { + "active_bytes_current": 66872599552, + "active_bytes_peak": 80934678528, + "allocated_bytes_current": 66872599552, + "allocated_bytes_peak": 80934678528, + "device": 0, + "inactive_split_bytes_current": 5181348864, + "inactive_split_bytes_peak": 5469035008, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98108964864, + "reserved_bytes_peak": 98108964864, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T12:04:04.664823Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872599552, + "allocated_bytes_current_delta": 31641144320, + "allocated_bytes_peak": 80934678528, + "allocated_bytes_peak_delta": 45468342272, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:02:54.695287Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5181348864, + "inactive_split_bytes_current_delta": 5046432768, + "inactive_split_bytes_peak": 5469035008, + "inactive_split_bytes_peak_delta": 5308403200, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98108964864, + "reserved_bytes_current_delta": 58862862336, + "reserved_bytes_peak": 98108964864, + "reserved_bytes_peak_delta": 58862862336, + "segment_count_current": 522, + "segment_count_current_delta": 450, + "segment_count_peak": 522, + "segment_count_peak_delta": 450, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T120404665310Z.pickle", + "timestamp_utc": "2026-08-01T12:04:04.775915Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T120404665310Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T120404665310Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f9c2cfc5cae68e2da9b31d0306666df88b0faa0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T120404665310Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e653a23f254900cba13db2dce0003066b87074ef3fabd638431f4ef77870dc2 +size 5397154 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T120405642759Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T120405642759Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4052658aa61a8108b48ccf56279addf8674f0751 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T120405642759Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872599552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104574373888, + "active_bytes_peak_delta": 23639695360, + "after": { + "active_bytes_current": 66872599552, + "active_bytes_peak": 104574373888, + "allocated_bytes_current": 66872599552, + "allocated_bytes_peak": 104574373888, + "device": 0, + "inactive_split_bytes_current": 5430909952, + "inactive_split_bytes_peak": 8050895872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108882034688, + "reserved_bytes_peak": 108882034688, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T12:04:05.642354Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872599552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104574373888, + "allocated_bytes_peak_delta": 23639695360, + "before": { + "active_bytes_current": 66872599552, + "active_bytes_peak": 80934678528, + "allocated_bytes_current": 66872599552, + "allocated_bytes_peak": 80934678528, + "device": 0, + "inactive_split_bytes_current": 5181348864, + "inactive_split_bytes_peak": 5469035008, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98108964864, + "reserved_bytes_peak": 98108964864, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T12:04:04.793904Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5430909952, + "inactive_split_bytes_current_delta": 249561088, + "inactive_split_bytes_peak": 8050895872, + "inactive_split_bytes_peak_delta": 2581860864, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108882034688, + "reserved_bytes_current_delta": 10773069824, + "reserved_bytes_peak": 108882034688, + "reserved_bytes_peak_delta": 10773069824, + "segment_count_current": 565, + "segment_count_current_delta": 43, + "segment_count_peak": 565, + "segment_count_peak_delta": 43, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T120405642759Z.pickle", + "timestamp_utc": "2026-08-01T12:04:05.744790Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T120405642759Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T120405642759Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..719a568ce8631423e0fdab400252cb4f986388ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T120405642759Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db52305e1101d233b772051f0df18bae8a4de891078f896e0473e697f41cce0a +size 5587257 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T120406578131Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T120406578131Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7c756736c30d04a475e6c07734cc87bc691c86db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T120406578131Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872599552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104574373888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66872599552, + "active_bytes_peak": 104574373888, + "allocated_bytes_current": 66872599552, + "allocated_bytes_peak": 104574373888, + "device": 0, + "inactive_split_bytes_current": 4963245056, + "inactive_split_bytes_peak": 8050895872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108882034688, + "reserved_bytes_peak": 108882034688, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T12:04:06.577711Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872599552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104574373888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66872599552, + "active_bytes_peak": 104574373888, + "allocated_bytes_current": 66872599552, + "allocated_bytes_peak": 104574373888, + "device": 0, + "inactive_split_bytes_current": 5430909952, + "inactive_split_bytes_peak": 8050895872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108882034688, + "reserved_bytes_peak": 108882034688, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T12:04:05.758211Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4963245056, + "inactive_split_bytes_current_delta": -467664896, + "inactive_split_bytes_peak": 8050895872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108882034688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108882034688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 565, + "segment_count_current_delta": 0, + "segment_count_peak": 565, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T120406578131Z.pickle", + "timestamp_utc": "2026-08-01T12:04:06.677744Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T120406578131Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T120406578131Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..90bd6d7e5fab81de3e46971255fa44306b07151f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T120406578131Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2ed41cbf3eb809f0521dea94bfcb5ea8bc3767535dae68fc46c00d227788dee +size 5585705 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T120409191107Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T120409191107Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ba56a0fb1189d5042a1e6cdf379a0e958d111bea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T120409191107Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889643008, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 156561311744, + "active_bytes_peak_delta": 51986937856, + "after": { + "active_bytes_current": 66889643008, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66889643008, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_peak": 11613522944, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_peak": 170894819328, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:09.190576Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889643008, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104588026880, + "allocated_bytes_peak_delta": 13652992, + "before": { + "active_bytes_current": 66872599552, + "active_bytes_peak": 104574373888, + "allocated_bytes_current": 66872599552, + "allocated_bytes_peak": 104574373888, + "device": 0, + "inactive_split_bytes_current": 4963245056, + "inactive_split_bytes_peak": 8050895872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108882034688, + "reserved_bytes_peak": 108882034688, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T12:04:06.690721Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_current_delta": 196866048, + "inactive_split_bytes_peak": 11613522944, + "inactive_split_bytes_peak_delta": 3562627072, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_current_delta": 62012784640, + "reserved_bytes_peak": 170894819328, + "reserved_bytes_peak_delta": 62012784640, + "segment_count_current": 928, + "segment_count_current_delta": 363, + "segment_count_peak": 928, + "segment_count_peak_delta": 363, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T120409191107Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.307769Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T120409191107Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T120409191107Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..857495dbe062b522634f52809737b61c21ad2fd5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T120409191107Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4e099dcff9ee3787549703ec631e5708f00bcb8be6e5f970cc328b23e54ceb2 +size 6595651 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T120409878022Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T120409878022Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c75aca2f6dc0d39b7f669c8c457c2d6c0927f68d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T120409878022Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889643008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156561311744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889643008, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66889643008, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_peak": 11613522944, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_peak": 170894819328, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:09.877533Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889643008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104588026880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889643008, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66889643008, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_peak": 11613522944, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_peak": 170894819328, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:09.322474Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11613522944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170894819328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T120409878022Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.885349Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T120409878022Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T120409878022Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c3a5069168deead563e5b7fa78cf23ef08dfb5c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T120409878022Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9cb7f3786921b337ee92548065138337884eea00314f24c2662771d30dd6875 +size 272653 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T120410434545Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T120410434545Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3c06ae0eff4125b58e5f4430404612c5db8c7694 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T120410434545Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889643008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156561311744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889643008, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66889643008, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_peak": 11613522944, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_peak": 170894819328, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:10.434108Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889643008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104588026880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889643008, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66889643008, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_peak": 11613522944, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_peak": 170894819328, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:09.886352Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11613522944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170894819328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T120410434545Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.441487Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T120410434545Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T120410434545Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..28d52169b84b1ed15d8bb8e4d6ff6dde7751d899 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T120410434545Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ff97b0251b06bc6d6ece4a32029792d3bbe30b6a7964702d825351d4be7598c +size 272653 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T120410988521Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T120410988521Z.json new file mode 100644 index 0000000000000000000000000000000000000000..df4afb9cb8cbc3a96b5bd0c67f5561413fd8b12e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T120410988521Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889643008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156561311744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889643008, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66889643008, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_peak": 11613522944, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_peak": 170894819328, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:10.988115Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889643008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104588026880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889643008, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66889643008, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_peak": 11613522944, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_peak": 170894819328, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:10.442483Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11613522944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170894819328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T120410988521Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.994955Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T120410988521Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T120410988521Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9557c325e5a94b7f6260b2f9698aa02be046bc4b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T120410988521Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e3ab8d0977f76ec6796a945ec423d783a53166c05b1a800b4a0b3e3c1d377c8 +size 272653 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T120411542374Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T120411542374Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7ec1144b31f4b7f344b5690bea9c3dd23f1f56f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T120411542374Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889643008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156561311744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889643008, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66889643008, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_peak": 11613522944, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_peak": 170894819328, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:11.541940Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889643008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104588026880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889643008, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66889643008, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_peak": 11613522944, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_peak": 170894819328, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:10.995979Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11613522944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170894819328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T120411542374Z.pickle", + "timestamp_utc": "2026-08-01T12:04:11.548841Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T120411542374Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T120411542374Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eb2525ef7a03a1bd5d19f2fa5c82137dbd2122d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T120411542374Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bb089913faf0c326028e0cf04e69441403ad2b121d0b54f80005056fffc07e6 +size 272653 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T120412099279Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T120412099279Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7c0b737a0412852bd5c5a7f5cb7d846a9569b4cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T120412099279Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889643008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156561311744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889643008, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66889643008, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_peak": 11613522944, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_peak": 170894819328, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:12.098864Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889643008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104588026880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889643008, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66889643008, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_peak": 11613522944, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_peak": 170894819328, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:11.549849Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11613522944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170894819328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T120412099279Z.pickle", + "timestamp_utc": "2026-08-01T12:04:12.105930Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T120412099279Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T120412099279Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7241ac78da88bd5813dbd43e5effde4dfd10355f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T120412099279Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d122bce96f198dfc5f1936aa3588a61a61679279c327394da2cc9fe39a6d1cc6 +size 272653 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T120458248547Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T120458248547Z.json new file mode 100644 index 0000000000000000000000000000000000000000..990755797fb86f48e875349019efb86e49d7dde6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T120458248547Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66856008704, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 156561311744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66856008704, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66856008704, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 4904338432, + "inactive_split_bytes_peak": 11613522944, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 172094390272, + "reserved_bytes_peak": 172094390272, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:04:58.245958Z", + "world_size": 8 + }, + "allocated_bytes_current": 66856008704, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104588026880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889643008, + "active_bytes_peak": 156561311744, + "allocated_bytes_current": 66889643008, + "allocated_bytes_peak": 104588026880, + "device": 0, + "inactive_split_bytes_current": 5160111104, + "inactive_split_bytes_peak": 11613522944, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 170894819328, + "reserved_bytes_peak": 170894819328, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:12.106901Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4904338432, + "inactive_split_bytes_current_delta": -255772672, + "inactive_split_bytes_peak": 11613522944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 172094390272, + "reserved_bytes_current_delta": 1199570944, + "reserved_bytes_peak": 172094390272, + "reserved_bytes_peak_delta": 1199570944, + "segment_count_current": 934, + "segment_count_current_delta": 6, + "segment_count_peak": 934, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T120458248547Z.pickle", + "timestamp_utc": "2026-08-01T12:04:58.334928Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T120458248547Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T120458248547Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f4307e7a50ac1d1dbf02beb9deda1cc3cfdb273d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T120458248547Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfaaad3e25b717498baca5252bb6cd7cf680dea4d6649dfc14ee9bb45a2356d3 +size 4091524 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T120404665354Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T120404665354Z.json new file mode 100644 index 0000000000000000000000000000000000000000..acf6d0ea7548f8ec82f8fc62d7291ed5bf9698a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T120404665354Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 31639102464, + "active_bytes_peak": 80939010048, + "active_bytes_peak_delta": 45472673792, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80939010048, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80939010048, + "device": 1, + "inactive_split_bytes_current": 5401494528, + "inactive_split_bytes_peak": 5723764736, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 97106526208, + "reserved_bytes_peak": 97106526208, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:04:04.664721Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 31639102464, + "allocated_bytes_peak": 80939010048, + "allocated_bytes_peak_delta": 45472673792, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:02:54.695751Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5401494528, + "inactive_split_bytes_current_delta": 5268675584, + "inactive_split_bytes_peak": 5723764736, + "inactive_split_bytes_peak_delta": 5563378688, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 97106526208, + "reserved_bytes_current_delta": 57862520832, + "reserved_bytes_peak": 97106526208, + "reserved_bytes_peak_delta": 57862520832, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T120404665354Z.pickle", + "timestamp_utc": "2026-08-01T12:04:04.777168Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T120404665354Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T120404665354Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb47890dc2f274c9ad92a044579e6aac4157eeb8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T120404665354Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24c47d2cd3696953717e34fe54c365877eba0602f0118de02539b788838798eb +size 5385789 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T120405642788Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T120405642788Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a359749b885f4b97c1db525309acd1dccd98d340 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T120405642788Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104570966016, + "active_bytes_peak_delta": 23631955968, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104570966016, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104570966016, + "device": 1, + "inactive_split_bytes_current": 5416174592, + "inactive_split_bytes_peak": 8007408640, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108728942592, + "reserved_bytes_peak": 108728942592, + "segment_count_current": 559, + "segment_count_peak": 559, + "timestamp_utc": "2026-08-01T12:04:05.642375Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104570966016, + "allocated_bytes_peak_delta": 23631955968, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80939010048, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80939010048, + "device": 1, + "inactive_split_bytes_current": 5401494528, + "inactive_split_bytes_peak": 5723764736, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 97106526208, + "reserved_bytes_peak": 97106526208, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:04:04.794775Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5416174592, + "inactive_split_bytes_current_delta": 14680064, + "inactive_split_bytes_peak": 8007408640, + "inactive_split_bytes_peak_delta": 2283643904, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108728942592, + "reserved_bytes_current_delta": 11622416384, + "reserved_bytes_peak": 108728942592, + "reserved_bytes_peak_delta": 11622416384, + "segment_count_current": 559, + "segment_count_current_delta": 47, + "segment_count_peak": 559, + "segment_count_peak_delta": 47, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T120405642788Z.pickle", + "timestamp_utc": "2026-08-01T12:04:05.747819Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T120405642788Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T120405642788Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..216633338ed59344d3e374c8acd9f2a2cd8c0ee4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T120405642788Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c12ff5298e7b9c6eeaaf7a275f94693b705a0b87ae66e0695f6284ee86eb68c4 +size 5589707 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T120406578092Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T120406578092Z.json new file mode 100644 index 0000000000000000000000000000000000000000..242b71a7a2df74385ed46de62e74edb81122a560 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T120406578092Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104571621376, + "active_bytes_peak_delta": 655360, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104571621376, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104571621376, + "device": 1, + "inactive_split_bytes_current": 5179196416, + "inactive_split_bytes_peak": 8007408640, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108728942592, + "reserved_bytes_peak": 108728942592, + "segment_count_current": 559, + "segment_count_peak": 559, + "timestamp_utc": "2026-08-01T12:04:06.577708Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104571621376, + "allocated_bytes_peak_delta": 655360, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104570966016, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104570966016, + "device": 1, + "inactive_split_bytes_current": 5416174592, + "inactive_split_bytes_peak": 8007408640, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108728942592, + "reserved_bytes_peak": 108728942592, + "segment_count_current": 559, + "segment_count_peak": 559, + "timestamp_utc": "2026-08-01T12:04:05.760972Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5179196416, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 8007408640, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108728942592, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108728942592, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 559, + "segment_count_current_delta": 0, + "segment_count_peak": 559, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T120406578092Z.pickle", + "timestamp_utc": "2026-08-01T12:04:06.681431Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T120406578092Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T120406578092Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d03b36f7255f22523c1a5d0f592040f1210ee922 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T120406578092Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eae90f8ac168a0792fe7c883f77328c4d48d1aafa800c117459fbdcbb1ae94a9 +size 5569565 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T120409191057Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T120409191057Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1df03d9d90661b72c8465907383240f1b15730ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T120409191057Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 156560973824, + "active_bytes_peak_delta": 51989352448, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 11679790080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.190542Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 14363648, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104571621376, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104571621376, + "device": 1, + "inactive_split_bytes_current": 5179196416, + "inactive_split_bytes_peak": 8007408640, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108728942592, + "reserved_bytes_peak": 108728942592, + "segment_count_current": 559, + "segment_count_peak": 559, + "timestamp_utc": "2026-08-01T12:04:06.694145Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_current_delta": 201060352, + "inactive_split_bytes_peak": 11679790080, + "inactive_split_bytes_peak_delta": 3672381440, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 62379786240, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 62379786240, + "segment_count_current": 927, + "segment_count_current_delta": 368, + "segment_count_peak": 927, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T120409191057Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.307768Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T120409191057Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T120409191057Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2c60cfe4beaf3fc2dbf9f48f3c250e3b99505bcf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T120409191057Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdca1589fcdcb83ac4ce9d0b4977b5be82b1eb4186417cb7eafa6f08bdae6734 +size 6594683 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T120409878021Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T120409878021Z.json new file mode 100644 index 0000000000000000000000000000000000000000..326e3f3cb557ae30d911d57de0a76ae369e32b46 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T120409878021Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156560973824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 11679790080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.877546Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 11679790080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.322142Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11679790080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T120409878021Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.885103Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T120409878021Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T120409878021Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6a3a753571beb2ea83db0dbb949b9c30efe62887 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T120409878021Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dc2d39060251171e5dea01897e5b36c6813a426c37f5d0b2dfd3e8740bb1506 +size 272517 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T120410434484Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T120410434484Z.json new file mode 100644 index 0000000000000000000000000000000000000000..58684160a0159ad384b05cbe0a2b7d90e8a6e9c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T120410434484Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156560973824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 11679790080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.434077Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 11679790080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.886059Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11679790080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T120410434484Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.441103Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T120410434484Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T120410434484Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..437cd4ec46ca3f4d762eca7e3bfbbb2d37a2b86f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T120410434484Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38a9015408d5d8fd45f571a9c4cf3aaa9023d68a4d943ba25212a55f5dc4cf93 +size 272517 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T120410988478Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T120410988478Z.json new file mode 100644 index 0000000000000000000000000000000000000000..98ca943bf21ec5962085b4aec98c89e9b6d2479e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T120410988478Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156560973824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 11679790080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.988085Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 11679790080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.442031Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11679790080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T120410988478Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.994636Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T120410988478Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T120410988478Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..02abd7619f33b4e4558f6427b19601e6c8c1cf22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T120410988478Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b95d2a8b1d8cb50fc3889f5c1f7fc7919e0af5e94c4dc1c46f0fd43ee1eda1c +size 272517 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T120411542075Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T120411542075Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ff9c2a58c1f6120abe7d8bcd55de359954b5be8b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T120411542075Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156560973824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 11679790080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:11.541691Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 11679790080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.995555Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11679790080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T120411542075Z.pickle", + "timestamp_utc": "2026-08-01T12:04:11.548233Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T120411542075Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T120411542075Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1473639a9cb90930daa13eb6f6b6c930d6244d2c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T120411542075Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a870e27b8b5c4c9d3d6a6c16eae9a82f5574858fa54a0b534d2fc721c04f4679 +size 272517 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T120412099202Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T120412099202Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd24477d1481836fddd7c6164472179f4cc5786a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T120412099202Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156560973824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 11679790080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:12.098802Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 11679790080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:11.549183Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11679790080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T120412099202Z.pickle", + "timestamp_utc": "2026-08-01T12:04:12.105303Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T120412099202Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T120412099202Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13496c329202415ee989da3f738dbbee829603c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T120412099202Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dac4e909274470e748f4247199b64eda3bfce845370198442a36f32514c26d2e +size 272517 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T120458246429Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T120458246429Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5e629b01f8d9bb53cc41dfbdee59bf4f58f290c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T120458246429Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853966848, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 156560973824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853966848, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66853966848, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5122386944, + "inactive_split_bytes_peak": 11679790080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 173012942848, + "reserved_bytes_peak": 173012942848, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:04:58.245949Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853966848, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156560973824, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 1, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 11679790080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:12.106262Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5122386944, + "inactive_split_bytes_current_delta": -257869824, + "inactive_split_bytes_peak": 11679790080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 173012942848, + "reserved_bytes_current_delta": 1904214016, + "reserved_bytes_peak": 173012942848, + "reserved_bytes_peak_delta": 1904214016, + "segment_count_current": 936, + "segment_count_current_delta": 9, + "segment_count_peak": 936, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T120458246429Z.pickle", + "timestamp_utc": "2026-08-01T12:04:58.329539Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T120458246429Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T120458246429Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f460140ebc1266c88f6017f5b848017fe9986c23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T120458246429Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e8f873f2574fc0952c2eb9ccddea6c8cda5e663f5990c8595a9e7ce7a7b7630 +size 3941912 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T120404665304Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T120404665304Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f36d01664151f52a838a569a5a312818c8c7f14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T120404665304Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 31639102464, + "active_bytes_peak": 80939010048, + "active_bytes_peak_delta": 45472673792, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80939010048, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80939010048, + "device": 2, + "inactive_split_bytes_current": 5401494528, + "inactive_split_bytes_peak": 5723764736, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 97576288256, + "reserved_bytes_peak": 97576288256, + "segment_count_current": 514, + "segment_count_peak": 514, + "timestamp_utc": "2026-08-01T12:04:04.664712Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 31639102464, + "allocated_bytes_peak": 80939010048, + "allocated_bytes_peak_delta": 45472673792, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:02:54.695716Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5401494528, + "inactive_split_bytes_current_delta": 5268675584, + "inactive_split_bytes_peak": 5723764736, + "inactive_split_bytes_peak_delta": 5563378688, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 97576288256, + "reserved_bytes_current_delta": 58332282880, + "reserved_bytes_peak": 97576288256, + "reserved_bytes_peak_delta": 58332282880, + "segment_count_current": 514, + "segment_count_current_delta": 443, + "segment_count_peak": 514, + "segment_count_peak_delta": 443, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T120404665304Z.pickle", + "timestamp_utc": "2026-08-01T12:04:04.775453Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T120404665304Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T120404665304Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..573bd00d3505828a29a6925a621ee416bf02e4ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T120404665304Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8dd1d003605e067ac452aa47f20ddd96c9a8259effa975faa4539a108b070c2 +size 5386138 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T120405642772Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T120405642772Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bbb79c436cb0b0c23759bb1a4e9dd30a4f14959d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T120405642772Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104569626112, + "active_bytes_peak_delta": 23630616064, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104569626112, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104569626112, + "device": 2, + "inactive_split_bytes_current": 5416174592, + "inactive_split_bytes_peak": 8101124608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109198704640, + "reserved_bytes_peak": 109198704640, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T12:04:05.642366Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104569626112, + "allocated_bytes_peak_delta": 23630616064, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80939010048, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80939010048, + "device": 2, + "inactive_split_bytes_current": 5401494528, + "inactive_split_bytes_peak": 5723764736, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 97576288256, + "reserved_bytes_peak": 97576288256, + "segment_count_current": 514, + "segment_count_peak": 514, + "timestamp_utc": "2026-08-01T12:04:04.793029Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5416174592, + "inactive_split_bytes_current_delta": 14680064, + "inactive_split_bytes_peak": 8101124608, + "inactive_split_bytes_peak_delta": 2377359872, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109198704640, + "reserved_bytes_current_delta": 11622416384, + "reserved_bytes_peak": 109198704640, + "reserved_bytes_peak_delta": 11622416384, + "segment_count_current": 561, + "segment_count_current_delta": 47, + "segment_count_peak": 561, + "segment_count_peak_delta": 47, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T120405642772Z.pickle", + "timestamp_utc": "2026-08-01T12:04:05.745056Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T120405642772Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T120405642772Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ac41b219c53f719501a8e7729865c3cf8d0bb5d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T120405642772Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbf68301e173dedb2ca3ec097f3c7901104719dd607ee98f013500273ccff0ba +size 5591269 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T120406578167Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T120406578167Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d61c0b3a93b097d50849ce3ce91b3588b843eef5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T120406578167Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104571621376, + "active_bytes_peak_delta": 1995264, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104571621376, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104571621376, + "device": 2, + "inactive_split_bytes_current": 5179196416, + "inactive_split_bytes_peak": 8101124608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109198704640, + "reserved_bytes_peak": 109198704640, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T12:04:06.577749Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104571621376, + "allocated_bytes_peak_delta": 1995264, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104569626112, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104569626112, + "device": 2, + "inactive_split_bytes_current": 5416174592, + "inactive_split_bytes_peak": 8101124608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109198704640, + "reserved_bytes_peak": 109198704640, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T12:04:05.758022Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5179196416, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 8101124608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109198704640, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109198704640, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T120406578167Z.pickle", + "timestamp_utc": "2026-08-01T12:04:06.678157Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T120406578167Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T120406578167Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..693bea00b46e72a4c38553b7221233985959fadc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T120406578167Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deadfaf77064351907dcec1fd68dc912b13969f7183aa7cc2d61037aa10abbf9 +size 5570495 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T120409191068Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T120409191068Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d660ed2baa0e0894b2ac0378c86a834d784330f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T120409191068Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 156558548992, + "active_bytes_peak_delta": 51986927616, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 12020211712, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.190548Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 15084544, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104571621376, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104571621376, + "device": 2, + "inactive_split_bytes_current": 5179196416, + "inactive_split_bytes_peak": 8101124608, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109198704640, + "reserved_bytes_peak": 109198704640, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T12:04:06.690785Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_current_delta": 201060352, + "inactive_split_bytes_peak": 12020211712, + "inactive_split_bytes_peak_delta": 3919087104, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 61910024192, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 61910024192, + "segment_count_current": 927, + "segment_count_current_delta": 366, + "segment_count_peak": 927, + "segment_count_peak_delta": 366, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T120409191068Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.309495Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T120409191068Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T120409191068Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..71af8c3a394a32514ffcbecf94ac3a133e49a81e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T120409191068Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fd010c03f0788b20eb7bd7c42a33aa2c24edf8ac797779428ca59f5fa37d0ab +size 6595597 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T120409877956Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T120409877956Z.json new file mode 100644 index 0000000000000000000000000000000000000000..555074308ea8774f8ce860060bc2e5d42ef1ba38 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T120409877956Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156558548992, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 12020211712, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.877505Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 12020211712, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.324126Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12020211712, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T120409877956Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.885171Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T120409877956Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T120409877956Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7a54d73424f805bf035a435cf6b377da45749920 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T120409877956Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24acb73ec491f4ec46d2d25a6f243ffce67d6bc7f0d1f99adb00179fb4b7ef60 +size 272351 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T120410434474Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T120410434474Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fba8c2673631a0f74fa3fefd0a6273d1cf898799 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T120410434474Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156558548992, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 12020211712, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.434067Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 12020211712, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.886135Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12020211712, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T120410434474Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.441294Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T120410434474Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T120410434474Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6fe856e03a66b268b929a0247b8d2a5985b6230d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T120410434474Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3de145a790a3d7c62bb2ff792eca1a566777ac557eff1e2caf1a28cdc7e49778 +size 272351 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T120410988471Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T120410988471Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d7759b946ff24d87b8b7916c9c91450f6f273720 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T120410988471Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156558548992, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 12020211712, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.988080Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 12020211712, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.442219Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12020211712, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T120410988471Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.994826Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T120410988471Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T120410988471Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3502c8c2ac11978c3bd177d1ade10aa85267b9fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T120410988471Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60291628955bc7450e9c2c35448caada1662f763301f7e9037355ae5281d7416 +size 272351 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T120411542047Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T120411542047Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ac560f62c948ed57d0b80bddda9c7bfff4dd76e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T120411542047Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156558548992, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 12020211712, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:11.541669Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 12020211712, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.995740Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12020211712, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T120411542047Z.pickle", + "timestamp_utc": "2026-08-01T12:04:11.548304Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T120411542047Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T120411542047Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f1497d1b0abfa368a5f296504fba9d06bf6574f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T120411542047Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccdcd6c113cb11872f192d96ebb08062495064942a3b7e0a7eab3168e17f13e8 +size 272351 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T120412099183Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T120412099183Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e0a53a1b7c9c65b89791f56716b192fca36d836d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T120412099183Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156558548992, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 12020211712, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:12.098798Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 12020211712, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:11.549291Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12020211712, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T120412099183Z.pickle", + "timestamp_utc": "2026-08-01T12:04:12.105275Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T120412099183Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T120412099183Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fbffac920d8cc31932c99eddb57f2546eed3e1bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T120412099183Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20a9c5d86342b2056a797dda4f1156788b9244512629c1ac4ae5e63b5372d811 +size 272351 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T120458246437Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T120458246437Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7df3a71769208135090fc0cf2fbaf2ad529b167f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T120458246437Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853966848, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 156558548992, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853966848, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66853966848, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5122386944, + "inactive_split_bytes_peak": 12020211712, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 173247823872, + "reserved_bytes_peak": 173247823872, + "segment_count_current": 937, + "segment_count_peak": 937, + "timestamp_utc": "2026-08-01T12:04:58.245959Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853966848, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156558548992, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 2, + "inactive_split_bytes_current": 5380256768, + "inactive_split_bytes_peak": 12020211712, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:12.106193Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5122386944, + "inactive_split_bytes_current_delta": -257869824, + "inactive_split_bytes_peak": 12020211712, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 173247823872, + "reserved_bytes_current_delta": 2139095040, + "reserved_bytes_peak": 173247823872, + "reserved_bytes_peak_delta": 2139095040, + "segment_count_current": 937, + "segment_count_current_delta": 10, + "segment_count_peak": 937, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T120458246437Z.pickle", + "timestamp_utc": "2026-08-01T12:04:58.336661Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T120458246437Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T120458246437Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a02cdb24ee8c969c059c0e6c921e0e323dd81f7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T120458246437Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f002bea11569ac3393fcf874e7aff5de7567089b30046b39ce41ada071d96523 +size 3942215 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T120404665371Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T120404665371Z.json new file mode 100644 index 0000000000000000000000000000000000000000..66a454a8b24dec51e0508cf2ffe65fadcbc51822 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T120404665371Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 31639102464, + "active_bytes_peak": 80932849664, + "active_bytes_peak_delta": 45466513408, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80932849664, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80932849664, + "device": 3, + "inactive_split_bytes_current": 5281956864, + "inactive_split_bytes_peak": 6170523648, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 98545172480, + "reserved_bytes_peak": 98545172480, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:04:04.664737Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 31639102464, + "allocated_bytes_peak": 80932849664, + "allocated_bytes_peak_delta": 45466513408, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:02:54.695608Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5281956864, + "inactive_split_bytes_current_delta": 5149137920, + "inactive_split_bytes_peak": 6170523648, + "inactive_split_bytes_peak_delta": 6010137600, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 98545172480, + "reserved_bytes_current_delta": 59301167104, + "reserved_bytes_peak": 98545172480, + "reserved_bytes_peak_delta": 59301167104, + "segment_count_current": 523, + "segment_count_current_delta": 452, + "segment_count_peak": 523, + "segment_count_peak_delta": 452, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T120404665371Z.pickle", + "timestamp_utc": "2026-08-01T12:04:04.772940Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T120404665371Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T120404665371Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aec068602c4a369258114eac3c16a6f7ab0d8e5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T120404665371Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:febc9e5dcde3ebc767c2cf2c9e4d9bd027a5b0481f8aed236ba175f34b25512c +size 5387597 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T120405642852Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T120405642852Z.json new file mode 100644 index 0000000000000000000000000000000000000000..63e4bba2d1d77de518b8092125671cead6efe996 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T120405642852Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104571707392, + "active_bytes_peak_delta": 23638857728, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104571707392, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104571707392, + "device": 3, + "inactive_split_bytes_current": 5059658752, + "inactive_split_bytes_peak": 8198818816, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 109548929024, + "reserved_bytes_peak": 109548929024, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T12:04:05.642442Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104571707392, + "allocated_bytes_peak_delta": 23638857728, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80932849664, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80932849664, + "device": 3, + "inactive_split_bytes_current": 5281956864, + "inactive_split_bytes_peak": 6170523648, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 98545172480, + "reserved_bytes_peak": 98545172480, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:04:04.789157Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5059658752, + "inactive_split_bytes_current_delta": -222298112, + "inactive_split_bytes_peak": 8198818816, + "inactive_split_bytes_peak_delta": 2028295168, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 109548929024, + "reserved_bytes_current_delta": 11003756544, + "reserved_bytes_peak": 109548929024, + "reserved_bytes_peak_delta": 11003756544, + "segment_count_current": 565, + "segment_count_current_delta": 42, + "segment_count_peak": 565, + "segment_count_peak_delta": 42, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T120405642852Z.pickle", + "timestamp_utc": "2026-08-01T12:04:05.741280Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T120405642852Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T120405642852Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f8904caf42c8fc73f1d52a2d9856ed81345b3bf6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T120405642852Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bb8ba38338dffe48fa46b597a0d9b18cfa43c57377703d325e9bcbb4a60f631 +size 5590824 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T120406578082Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T120406578082Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb0999ab87e35915e46abf180eed4757960a1cf3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T120406578082Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104576403456, + "active_bytes_peak_delta": 4696064, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104576403456, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104576403456, + "device": 3, + "inactive_split_bytes_current": 5061755904, + "inactive_split_bytes_peak": 8198818816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 109548929024, + "reserved_bytes_peak": 109548929024, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T12:04:06.577691Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104576403456, + "allocated_bytes_peak_delta": 4696064, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104571707392, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104571707392, + "device": 3, + "inactive_split_bytes_current": 5059658752, + "inactive_split_bytes_peak": 8198818816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 109548929024, + "reserved_bytes_peak": 109548929024, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T12:04:05.752468Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5061755904, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 8198818816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 109548929024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109548929024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 565, + "segment_count_current_delta": 0, + "segment_count_peak": 565, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T120406578082Z.pickle", + "timestamp_utc": "2026-08-01T12:04:06.677712Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T120406578082Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T120406578082Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a80e1368147682895fd6b1df5dbfdffed8f1484 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T120406578082Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29a6cbfab520ec7f45633859467251739c1f1180d20bb0dd6cf5333214bf61f7 +size 5584011 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T120409191026Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T120409191026Z.json new file mode 100644 index 0000000000000000000000000000000000000000..409d8a4f94f9cac531bfda7c1be2efa3912176f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T120409191026Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 156559073280, + "active_bytes_peak_delta": 51982669824, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_peak": 11469608448, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 170991288320, + "reserved_bytes_peak": 170991288320, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:09.190538Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 9581568, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104576403456, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104576403456, + "device": 3, + "inactive_split_bytes_current": 5061755904, + "inactive_split_bytes_peak": 8198818816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 109548929024, + "reserved_bytes_peak": 109548929024, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T12:04:06.694778Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_current_delta": 196866048, + "inactive_split_bytes_peak": 11469608448, + "inactive_split_bytes_peak_delta": 3270789632, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 170991288320, + "reserved_bytes_current_delta": 61442359296, + "reserved_bytes_peak": 170991288320, + "reserved_bytes_peak_delta": 61442359296, + "segment_count_current": 928, + "segment_count_current_delta": 363, + "segment_count_peak": 928, + "segment_count_peak_delta": 363, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T120409191026Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.304140Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T120409191026Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T120409191026Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aac62b2539ee1828552d4b07fe537d40b09c9e9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T120409191026Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:974525f1481732e759a29902c76ccdb7a7344bdbaa06f55aafa2e85e3fdd4fae +size 6594446 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T120409877953Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T120409877953Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e2367a35821c93b5474f22dbb14ebc62b87793f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T120409877953Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156559073280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_peak": 11469608448, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 170991288320, + "reserved_bytes_peak": 170991288320, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:09.877507Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_peak": 11469608448, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 170991288320, + "reserved_bytes_peak": 170991288320, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:09.317303Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11469608448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 170991288320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170991288320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T120409877953Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.884528Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T120409877953Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T120409877953Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3cb2a9440c321204a1ed9433e090a3cea237aaf0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T120409877953Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5c21cf18c2aa9c9dec77d84a7817982f818f1458da25b1059d2b4a437fa4892 +size 272305 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T120410434472Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T120410434472Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c7fd829fa9fd432facf6e845855821f9f331ec06 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T120410434472Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156559073280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_peak": 11469608448, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 170991288320, + "reserved_bytes_peak": 170991288320, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:10.434064Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_peak": 11469608448, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 170991288320, + "reserved_bytes_peak": 170991288320, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:09.885542Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11469608448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 170991288320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170991288320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T120410434472Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.441036Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T120410434472Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T120410434472Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..52206c0240bbe68f1ce38c778d28bf3b4d0eb92d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T120410434472Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19fdcd78759bec164f61be1fcfdb9a4e95187e02f2c7d31612ce712b61b894b3 +size 272305 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T120410988486Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T120410988486Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f70b459dd0f16d4dbde854bf4991820e1fb60ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T120410988486Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156559073280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_peak": 11469608448, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 170991288320, + "reserved_bytes_peak": 170991288320, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:10.988095Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_peak": 11469608448, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 170991288320, + "reserved_bytes_peak": 170991288320, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:10.441973Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11469608448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 170991288320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170991288320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T120410988486Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.994560Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T120410988486Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T120410988486Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ce3617de6d2f95b0b710f59b1c8e4ee6ca7f9ee1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T120410988486Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e942c48967a42439205c75eaece0f6c03a956ec1351d78553fc6a2cfab68ff8 +size 272305 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T120411542082Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T120411542082Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7efb876c6230705864dc6edea327021ac80f85af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T120411542082Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156559073280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_peak": 11469608448, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 170991288320, + "reserved_bytes_peak": 170991288320, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:11.541678Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_peak": 11469608448, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 170991288320, + "reserved_bytes_peak": 170991288320, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:10.995486Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11469608448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 170991288320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170991288320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T120411542082Z.pickle", + "timestamp_utc": "2026-08-01T12:04:11.548214Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T120411542082Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T120411542082Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..407387b72813848ab28ae99f11069e1686e272b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T120411542082Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:356d066fbc113329ceb6d92ac272d21341bdb58cd0c667d1a297efc8c0d0db09 +size 272305 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T120412099209Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T120412099209Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ffadb19384b94bd2cb24baec858115a87b997936 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T120412099209Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156559073280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_peak": 11469608448, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 170991288320, + "reserved_bytes_peak": 170991288320, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:12.098819Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_peak": 11469608448, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 170991288320, + "reserved_bytes_peak": 170991288320, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:11.549168Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11469608448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 170991288320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170991288320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T120412099209Z.pickle", + "timestamp_utc": "2026-08-01T12:04:12.105342Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T120412099209Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T120412099209Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3607f6cccd2a804b822841f8d50296358efd40e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T120412099209Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9d051dafb039ac2cf23bd764ce0353074594832d7a466742118a4d3d772d758 +size 272305 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T120458246433Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T120458246433Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6decd2e173f39d802b738020a19426b970ed686 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T120458246433Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853966848, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 156559073280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853966848, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66853966848, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5002849280, + "inactive_split_bytes_peak": 11469608448, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 172660621312, + "reserved_bytes_peak": 172660621312, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:04:58.245946Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853966848, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104585985024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156559073280, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104585985024, + "device": 3, + "inactive_split_bytes_current": 5258621952, + "inactive_split_bytes_peak": 11469608448, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 170991288320, + "reserved_bytes_peak": 170991288320, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:12.106263Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5002849280, + "inactive_split_bytes_current_delta": -255772672, + "inactive_split_bytes_peak": 11469608448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 172660621312, + "reserved_bytes_current_delta": 1669332992, + "reserved_bytes_peak": 172660621312, + "reserved_bytes_peak_delta": 1669332992, + "segment_count_current": 936, + "segment_count_current_delta": 8, + "segment_count_peak": 936, + "segment_count_peak_delta": 8, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T120458246433Z.pickle", + "timestamp_utc": "2026-08-01T12:04:58.329895Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T120458246433Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T120458246433Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5b573dc39f9d244f12c8950d75f534e7ec8ad1d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T120458246433Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea7920ca93da37f00ad750f3f8f2518d43cf02a4c854ee988e66c8e5dcbbf707 +size 3941830 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T120404665157Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T120404665157Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b9ed3fb1d31f55196c25d3491c2a05e0dc28be33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T120404665157Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871777280, + "active_bytes_current_delta": 31639879680, + "active_bytes_peak": 80935405568, + "active_bytes_peak_delta": 45468626944, + "after": { + "active_bytes_current": 66871777280, + "active_bytes_peak": 80935405568, + "allocated_bytes_current": 66871777280, + "allocated_bytes_peak": 80935405568, + "device": 4, + "inactive_split_bytes_current": 5106673664, + "inactive_split_bytes_peak": 5820473856, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 98545172480, + "reserved_bytes_peak": 98545172480, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:04:04.664713Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871777280, + "allocated_bytes_current_delta": 31639879680, + "allocated_bytes_peak": 80935405568, + "allocated_bytes_peak_delta": 45468626944, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:02:54.695389Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5106673664, + "inactive_split_bytes_current_delta": 4974297088, + "inactive_split_bytes_peak": 5820473856, + "inactive_split_bytes_peak_delta": 5667064320, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 98545172480, + "reserved_bytes_current_delta": 59301167104, + "reserved_bytes_peak": 98545172480, + "reserved_bytes_peak_delta": 59301167104, + "segment_count_current": 523, + "segment_count_current_delta": 452, + "segment_count_peak": 523, + "segment_count_peak_delta": 452, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T120404665157Z.pickle", + "timestamp_utc": "2026-08-01T12:04:04.776023Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T120404665157Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T120404665157Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0b10bcba97369465d67705ae79cdfe8c920c42ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T120404665157Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d13aa62c784a7b6db41d4f8d4195098be87f52d273dbb3c50b71f3f231e83ef +size 5397628 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T120405642758Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T120405642758Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e0395eb8eb826e8c371fae47407dc5f5c09645e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T120405642758Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871777280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104564740608, + "active_bytes_peak_delta": 23629335040, + "after": { + "active_bytes_current": 66871777280, + "active_bytes_peak": 104564740608, + "allocated_bytes_current": 66871777280, + "allocated_bytes_peak": 104564740608, + "device": 4, + "inactive_split_bytes_current": 5117159424, + "inactive_split_bytes_peak": 8285316608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109184024576, + "reserved_bytes_peak": 109184024576, + "segment_count_current": 562, + "segment_count_peak": 562, + "timestamp_utc": "2026-08-01T12:04:05.642343Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871777280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104564740608, + "allocated_bytes_peak_delta": 23629335040, + "before": { + "active_bytes_current": 66871777280, + "active_bytes_peak": 80935405568, + "allocated_bytes_current": 66871777280, + "allocated_bytes_peak": 80935405568, + "device": 4, + "inactive_split_bytes_current": 5106673664, + "inactive_split_bytes_peak": 5820473856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 98545172480, + "reserved_bytes_peak": 98545172480, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:04:04.795350Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5117159424, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 8285316608, + "inactive_split_bytes_peak_delta": 2464842752, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109184024576, + "reserved_bytes_current_delta": 10638852096, + "reserved_bytes_peak": 109184024576, + "reserved_bytes_peak_delta": 10638852096, + "segment_count_current": 562, + "segment_count_current_delta": 39, + "segment_count_peak": 562, + "segment_count_peak_delta": 39, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T120405642758Z.pickle", + "timestamp_utc": "2026-08-01T12:04:05.748574Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T120405642758Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T120405642758Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e593e09d30a5a4dde78dc9f85191af5386d2f99 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T120405642758Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e503b17847eda805f2cadb34df7d6f01f6d838a5804d78b09a3247a7a2fb307 +size 5595900 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T120406578084Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T120406578084Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f98d3563cbddc7acee6cb1acc40246f1599d9846 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T120406578084Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871777280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104565789184, + "active_bytes_peak_delta": 1048576, + "after": { + "active_bytes_current": 66871777280, + "active_bytes_peak": 104565789184, + "allocated_bytes_current": 66871777280, + "allocated_bytes_peak": 104565789184, + "device": 4, + "inactive_split_bytes_current": 4886472704, + "inactive_split_bytes_peak": 8285316608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109184024576, + "reserved_bytes_peak": 109184024576, + "segment_count_current": 562, + "segment_count_peak": 562, + "timestamp_utc": "2026-08-01T12:04:06.577685Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871777280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104565789184, + "allocated_bytes_peak_delta": 1048576, + "before": { + "active_bytes_current": 66871777280, + "active_bytes_peak": 104564740608, + "allocated_bytes_current": 66871777280, + "allocated_bytes_peak": 104564740608, + "device": 4, + "inactive_split_bytes_current": 5117159424, + "inactive_split_bytes_peak": 8285316608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 109184024576, + "reserved_bytes_peak": 109184024576, + "segment_count_current": 562, + "segment_count_peak": 562, + "timestamp_utc": "2026-08-01T12:04:05.763619Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4886472704, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 8285316608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109184024576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109184024576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 562, + "segment_count_current_delta": 0, + "segment_count_peak": 562, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T120406578084Z.pickle", + "timestamp_utc": "2026-08-01T12:04:06.682073Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T120406578084Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T120406578084Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..52aa7a5d36965ffad01b72268e467191c53f1512 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T120406578084Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a06dfb2b9a990d370a66ab43241efcd172e91c0447d0bf0f7fae248845fdb7a +size 5584664 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T120409191092Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T120409191092Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bbfcd941857e3c90925d2e500c680d325b522812 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T120409191092Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888820736, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 156554722304, + "active_bytes_peak_delta": 51988933120, + "after": { + "active_bytes_current": 66888820736, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66888820736, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_peak": 11293512192, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 170817224704, + "reserved_bytes_peak": 170817224704, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.190542Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888820736, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104587204608, + "allocated_bytes_peak_delta": 21415424, + "before": { + "active_bytes_current": 66871777280, + "active_bytes_peak": 104565789184, + "allocated_bytes_current": 66871777280, + "allocated_bytes_peak": 104565789184, + "device": 4, + "inactive_split_bytes_current": 4886472704, + "inactive_split_bytes_peak": 8285316608, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 109184024576, + "reserved_bytes_peak": 109184024576, + "segment_count_current": 562, + "segment_count_peak": 562, + "timestamp_utc": "2026-08-01T12:04:06.696929Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_current_delta": 196866048, + "inactive_split_bytes_peak": 11293512192, + "inactive_split_bytes_peak_delta": 3008195584, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 170817224704, + "reserved_bytes_current_delta": 61633200128, + "reserved_bytes_peak": 170817224704, + "reserved_bytes_peak_delta": 61633200128, + "segment_count_current": 927, + "segment_count_current_delta": 365, + "segment_count_peak": 927, + "segment_count_peak_delta": 365, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T120409191092Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.312885Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T120409191092Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T120409191092Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a97715a36144d13e7da2d00feeb170eb146e0fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T120409191092Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe85f44655485563dba57a66b4d253e9272cf8f52fa145af37f4ad44eb14f9b9 +size 6594525 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T120409877985Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T120409877985Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee1275615ba48fc56e1edc30227bde670d9628db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T120409877985Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888820736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156554722304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888820736, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66888820736, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_peak": 11293512192, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 170817224704, + "reserved_bytes_peak": 170817224704, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.877519Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888820736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104587204608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888820736, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66888820736, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_peak": 11293512192, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 170817224704, + "reserved_bytes_peak": 170817224704, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.330291Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11293512192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 170817224704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170817224704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T120409877985Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.885566Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T120409877985Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T120409877985Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b067f20a9712afb00b91a85507bf14a227e815dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T120409877985Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c53781d2f91f3035ecf60f25474e81c7e41e010c7250202713e93d314159ebb +size 273454 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T120410434483Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T120410434483Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae001a3dce02c1d572f1307f5bb90f7848ea1056 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T120410434483Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888820736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156554722304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888820736, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66888820736, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_peak": 11293512192, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 170817224704, + "reserved_bytes_peak": 170817224704, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.434066Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888820736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104587204608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888820736, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66888820736, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_peak": 11293512192, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 170817224704, + "reserved_bytes_peak": 170817224704, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.886550Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11293512192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 170817224704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170817224704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T120410434483Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.441423Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T120410434483Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T120410434483Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..621c0542618737bcf18231439bf8b26b6b1eb557 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T120410434483Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d282971f4121b0ff007a5f90cbf9388aa4f5212171038a5dfe6916e4b98e3619 +size 273454 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T120410988491Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T120410988491Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1463bb0f62619468895c628e00fe12dc59eb9060 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T120410988491Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888820736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156554722304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888820736, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66888820736, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_peak": 11293512192, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 170817224704, + "reserved_bytes_peak": 170817224704, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.988088Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888820736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104587204608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888820736, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66888820736, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_peak": 11293512192, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 170817224704, + "reserved_bytes_peak": 170817224704, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.442395Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11293512192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 170817224704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170817224704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T120410988491Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.995057Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T120410988491Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T120410988491Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a99101e648380c9b4354a24d97851861cbdacbba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T120410988491Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:257a45cfb0bbfc1abe5a042af6a1b0f6a6a5b8a6b579b166b9af779a150166aa +size 273454 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T120411542083Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T120411542083Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2c7d5c91e29b6c128bb686f7f8c351a46637061e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T120411542083Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888820736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156554722304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888820736, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66888820736, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_peak": 11293512192, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 170817224704, + "reserved_bytes_peak": 170817224704, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:11.541688Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888820736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104587204608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888820736, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66888820736, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_peak": 11293512192, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 170817224704, + "reserved_bytes_peak": 170817224704, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.995993Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11293512192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 170817224704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170817224704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T120411542083Z.pickle", + "timestamp_utc": "2026-08-01T12:04:11.548473Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T120411542083Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T120411542083Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3bd5b418f7cc1d48093d66ca03be8118d9e3a979 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T120411542083Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df9148af0d80b21f23b885bdbed392a4fe85b9758b9b2370d3ae4516157ad224 +size 273454 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T120412099181Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T120412099181Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6be5d428a6ddfbb0bb59ea7c3bc811e318f529a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T120412099181Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888820736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156554722304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888820736, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66888820736, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_peak": 11293512192, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 170817224704, + "reserved_bytes_peak": 170817224704, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:12.098789Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888820736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104587204608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888820736, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66888820736, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_peak": 11293512192, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 170817224704, + "reserved_bytes_peak": 170817224704, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:11.549443Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11293512192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 170817224704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170817224704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T120412099181Z.pickle", + "timestamp_utc": "2026-08-01T12:04:12.105539Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T120412099181Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T120412099181Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4317df4360e75ffb5841224e030beeceada9c1ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T120412099181Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97ee090a6bdc421cba69ffd02d8501c8f9e61b494809c8d5f21d1d1def158c76 +size 273454 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T120458246451Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T120458246451Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ac01ee4f3845dcadf54a0f2714b11ffa34005eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T120458246451Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855186432, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 156554722304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855186432, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66855186432, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 4827566080, + "inactive_split_bytes_peak": 11293512192, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 172016795648, + "reserved_bytes_peak": 172016795648, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:04:58.245967Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855186432, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104587204608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888820736, + "active_bytes_peak": 156554722304, + "allocated_bytes_current": 66888820736, + "allocated_bytes_peak": 104587204608, + "device": 4, + "inactive_split_bytes_current": 5083338752, + "inactive_split_bytes_peak": 11293512192, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 170817224704, + "reserved_bytes_peak": 170817224704, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:12.106474Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4827566080, + "inactive_split_bytes_current_delta": -255772672, + "inactive_split_bytes_peak": 11293512192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 172016795648, + "reserved_bytes_current_delta": 1199570944, + "reserved_bytes_peak": 172016795648, + "reserved_bytes_peak_delta": 1199570944, + "segment_count_current": 933, + "segment_count_current_delta": 6, + "segment_count_peak": 933, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T120458246451Z.pickle", + "timestamp_utc": "2026-08-01T12:04:58.336578Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T120458246451Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T120458246451Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a8e32497bca95027817b7fb911c7e98bf4780cf4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T120458246451Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e06da348da86ff5089b43dabf8e1eac8a23bd704032ac3d5164803b1e7f3faf4 +size 3957791 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T120404665524Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T120404665524Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f8a596712ff6b0538e22fc914df4f2952aa7b8a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T120404665524Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 31639102464, + "active_bytes_peak": 80939010048, + "active_bytes_peak_delta": 45472673792, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80939010048, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80939010048, + "device": 5, + "inactive_split_bytes_current": 5401494528, + "inactive_split_bytes_peak": 5723764736, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 97576288256, + "reserved_bytes_peak": 97576288256, + "segment_count_current": 514, + "segment_count_peak": 514, + "timestamp_utc": "2026-08-01T12:04:04.664871Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 31639102464, + "allocated_bytes_peak": 80939010048, + "allocated_bytes_peak_delta": 45472673792, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:02:54.695584Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5401494528, + "inactive_split_bytes_current_delta": 5268675584, + "inactive_split_bytes_peak": 5723764736, + "inactive_split_bytes_peak_delta": 5563378688, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 97576288256, + "reserved_bytes_current_delta": 58332282880, + "reserved_bytes_peak": 97576288256, + "reserved_bytes_peak_delta": 58332282880, + "segment_count_current": 514, + "segment_count_current_delta": 443, + "segment_count_peak": 514, + "segment_count_peak_delta": 443, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T120404665524Z.pickle", + "timestamp_utc": "2026-08-01T12:04:04.782284Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T120404665524Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T120404665524Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b644e68cb13adcdb6b8743205b1260110f522134 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T120404665524Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42f7fe780fc999f5eaea29c26927bf39492c9a5023d62c0955b963a97d9d8b8e +size 5386143 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T120405642808Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T120405642808Z.json new file mode 100644 index 0000000000000000000000000000000000000000..58be08afa8b94600627b146ce7f130543d68b202 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T120405642808Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104569626112, + "active_bytes_peak_delta": 23630616064, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104569626112, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104569626112, + "device": 5, + "inactive_split_bytes_current": 5183390720, + "inactive_split_bytes_peak": 8101124608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108728942592, + "reserved_bytes_peak": 108728942592, + "segment_count_current": 559, + "segment_count_peak": 559, + "timestamp_utc": "2026-08-01T12:04:05.642387Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104569626112, + "allocated_bytes_peak_delta": 23630616064, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80939010048, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80939010048, + "device": 5, + "inactive_split_bytes_current": 5401494528, + "inactive_split_bytes_peak": 5723764736, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 97576288256, + "reserved_bytes_peak": 97576288256, + "segment_count_current": 514, + "segment_count_peak": 514, + "timestamp_utc": "2026-08-01T12:04:04.801924Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5183390720, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 8101124608, + "inactive_split_bytes_peak_delta": 2377359872, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108728942592, + "reserved_bytes_current_delta": 11152654336, + "reserved_bytes_peak": 108728942592, + "reserved_bytes_peak_delta": 11152654336, + "segment_count_current": 559, + "segment_count_current_delta": 45, + "segment_count_peak": 559, + "segment_count_peak_delta": 45, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T120405642808Z.pickle", + "timestamp_utc": "2026-08-01T12:04:05.747225Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T120405642808Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T120405642808Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f5a8fff70f2f91b4a65464f3eddf2164e3532a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T120405642808Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9589637aab7ccf113373a516869cc4406fcb0e006b55bf97402fd525097b863b +size 5590184 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T120406578095Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T120406578095Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f921384cac575922deddadb558320ac5479fad9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T120406578095Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104571621376, + "active_bytes_peak_delta": 1995264, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104571621376, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104571621376, + "device": 5, + "inactive_split_bytes_current": 5414077440, + "inactive_split_bytes_peak": 8101124608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108728942592, + "reserved_bytes_peak": 108728942592, + "segment_count_current": 559, + "segment_count_peak": 559, + "timestamp_utc": "2026-08-01T12:04:06.577691Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104571621376, + "allocated_bytes_peak_delta": 1995264, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104569626112, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104569626112, + "device": 5, + "inactive_split_bytes_current": 5183390720, + "inactive_split_bytes_peak": 8101124608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108728942592, + "reserved_bytes_peak": 108728942592, + "segment_count_current": 559, + "segment_count_peak": 559, + "timestamp_utc": "2026-08-01T12:04:05.762455Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5414077440, + "inactive_split_bytes_current_delta": 230686720, + "inactive_split_bytes_peak": 8101124608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108728942592, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108728942592, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 559, + "segment_count_current_delta": 0, + "segment_count_peak": 559, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T120406578095Z.pickle", + "timestamp_utc": "2026-08-01T12:04:06.682187Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T120406578095Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T120406578095Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60a9951d5075c73c896e6152fbef16725f068084 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T120406578095Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cdac4c116bd33f722658500d3d8fd8ba6ffe81cf430aeeae1a8fc89534c824a +size 5570118 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T120409191080Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T120409191080Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5124cd3f24c8713d72b85b9cd5d4468d60a57459 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T120409191080Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 156557697024, + "active_bytes_peak_delta": 51986075648, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_peak": 11993773056, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 171343609856, + "reserved_bytes_peak": 171343609856, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:09.190554Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 15084544, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104571621376, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104571621376, + "device": 5, + "inactive_split_bytes_current": 5414077440, + "inactive_split_bytes_peak": 8101124608, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108728942592, + "reserved_bytes_peak": 108728942592, + "segment_count_current": 559, + "segment_count_peak": 559, + "timestamp_utc": "2026-08-01T12:04:06.697075Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_current_delta": -35917824, + "inactive_split_bytes_peak": 11993773056, + "inactive_split_bytes_peak_delta": 3892648448, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 171343609856, + "reserved_bytes_current_delta": 62614667264, + "reserved_bytes_peak": 171343609856, + "reserved_bytes_peak_delta": 62614667264, + "segment_count_current": 928, + "segment_count_current_delta": 369, + "segment_count_peak": 928, + "segment_count_peak_delta": 369, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T120409191080Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.314568Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T120409191080Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T120409191080Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03285b523e9b466a379a7b4ec8c9a9a6a1932e57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T120409191080Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9017b00b643bd6d24a0f70aeb701f2c54c84ad1b0fd8b80324e22e325c237b6 +size 6595267 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T120409878078Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T120409878078Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d36045216a449cd28be63ac857501d96172c01a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T120409878078Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156557697024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_peak": 11993773056, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 171343609856, + "reserved_bytes_peak": 171343609856, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:09.877563Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_peak": 11993773056, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 171343609856, + "reserved_bytes_peak": 171343609856, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:09.331200Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11993773056, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 171343609856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171343609856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T120409878078Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.885981Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T120409878078Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T120409878078Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b8184dc97906b52461b3a0fc50e55df8f94a456 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T120409878078Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5e4c2a412668999cda4d92998f0cc48aa36035f774950dfa5e124eb6f0a0689 +size 272625 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T120410434483Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T120410434483Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2b74857a88a269869a9f2352322941cc7626adc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T120410434483Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156557697024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_peak": 11993773056, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 171343609856, + "reserved_bytes_peak": 171343609856, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:10.434071Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_peak": 11993773056, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 171343609856, + "reserved_bytes_peak": 171343609856, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:09.887007Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11993773056, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 171343609856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171343609856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T120410434483Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.441401Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T120410434483Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T120410434483Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1327e245fa2354c11c3a05c6fccf3c9199ae4fff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T120410434483Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9438828ae801fbe60bc0eeee31ec2565555da169e216fe8a3ee83d5754d06f66 +size 272625 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T120410988526Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T120410988526Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4997d0a858a35a61cecb522a3f373315663e9fc8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T120410988526Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156557697024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_peak": 11993773056, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 171343609856, + "reserved_bytes_peak": 171343609856, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:10.988126Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_peak": 11993773056, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 171343609856, + "reserved_bytes_peak": 171343609856, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:10.442383Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11993773056, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 171343609856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171343609856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T120410988526Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.994933Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T120410988526Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T120410988526Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e73dad0beec8e46c66a27e78aec3e1904b31cc32 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T120410988526Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69361c45f891f97f071cf1b263403d7131a22b100150a5c857af8b81532cb8fc +size 272625 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T120411542097Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T120411542097Z.json new file mode 100644 index 0000000000000000000000000000000000000000..21a6b56e4d746f65212b3aaeb9e7e3fa7d148e6e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T120411542097Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156557697024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_peak": 11993773056, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 171343609856, + "reserved_bytes_peak": 171343609856, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:11.541700Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_peak": 11993773056, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 171343609856, + "reserved_bytes_peak": 171343609856, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:10.995924Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11993773056, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 171343609856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171343609856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T120411542097Z.pickle", + "timestamp_utc": "2026-08-01T12:04:11.548525Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T120411542097Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T120411542097Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fdce64719c81963988230e006a8ce5670710d6be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T120411542097Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04d34f67e5813ad0e2684a428efb683d908475ac1fdc4068aea08611e0291b59 +size 272625 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T120412099213Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T120412099213Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e62a95ea6c78310c53b598de6818cb096951da59 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T120412099213Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156557697024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_peak": 11993773056, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 171343609856, + "reserved_bytes_peak": 171343609856, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:12.098805Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_peak": 11993773056, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 171343609856, + "reserved_bytes_peak": 171343609856, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:11.549517Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11993773056, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 171343609856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171343609856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T120412099213Z.pickle", + "timestamp_utc": "2026-08-01T12:04:12.105430Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T120412099213Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T120412099213Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c753cdb8c72674a0512c9ff0205a210db3babed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T120412099213Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8642d6205ae337b09ebba85c110d6f70a854ce9fa5c2dff6366cf0950b238144 +size 272625 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T120458246432Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T120458246432Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b85c988ab8e37b1c1e3f6aa5c14ae8fe9bfbc705 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T120458246432Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853966848, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 156557697024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853966848, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66853966848, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5122386944, + "inactive_split_bytes_peak": 11993773056, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 172308299776, + "reserved_bytes_peak": 172308299776, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:04:58.245966Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853966848, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104586705920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156557697024, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586705920, + "device": 5, + "inactive_split_bytes_current": 5378159616, + "inactive_split_bytes_peak": 11993773056, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 171343609856, + "reserved_bytes_peak": 171343609856, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:04:12.106408Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5122386944, + "inactive_split_bytes_current_delta": -255772672, + "inactive_split_bytes_peak": 11993773056, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 172308299776, + "reserved_bytes_current_delta": 964689920, + "reserved_bytes_peak": 172308299776, + "reserved_bytes_peak_delta": 964689920, + "segment_count_current": 933, + "segment_count_current_delta": 5, + "segment_count_peak": 933, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T120458246432Z.pickle", + "timestamp_utc": "2026-08-01T12:04:58.329356Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T120458246432Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T120458246432Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..33fa4ecabbdd4e95509cf8794e3f6de622126552 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T120458246432Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1162ac7ecc04d3680d131286e1b6e2468ea0db3f8a06251c5cd13f725795086 +size 3955191 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T120404665401Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T120404665401Z.json new file mode 100644 index 0000000000000000000000000000000000000000..233dbb4a85b1ccac8c7a427405613749755145be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T120404665401Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 31639102464, + "active_bytes_peak": 80935405568, + "active_bytes_peak_delta": 45469069312, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80935405568, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80935405568, + "device": 6, + "inactive_split_bytes_current": 5281956864, + "inactive_split_bytes_peak": 5390814720, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98075410432, + "reserved_bytes_peak": 98075410432, + "segment_count_current": 521, + "segment_count_peak": 521, + "timestamp_utc": "2026-08-01T12:04:04.664732Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 31639102464, + "allocated_bytes_peak": 80935405568, + "allocated_bytes_peak_delta": 45469069312, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:02:54.695594Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5281956864, + "inactive_split_bytes_current_delta": 5149137920, + "inactive_split_bytes_peak": 5390814720, + "inactive_split_bytes_peak_delta": 5230428672, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98075410432, + "reserved_bytes_current_delta": 58831405056, + "reserved_bytes_peak": 98075410432, + "reserved_bytes_peak_delta": 58831405056, + "segment_count_current": 521, + "segment_count_current_delta": 450, + "segment_count_peak": 521, + "segment_count_peak_delta": 450, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T120404665401Z.pickle", + "timestamp_utc": "2026-08-01T12:04:04.781760Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T120404665401Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T120404665401Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f035bbeea91501f384bfaea9ebdbbf1f7e9ec451 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T120404665401Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9606b465b3ec933506f6b52756ccc37e2e23424f1cbac82125a34e28a17e8eb4 +size 5387240 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T120405642786Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T120405642786Z.json new file mode 100644 index 0000000000000000000000000000000000000000..672b5c0d5f6ced555786118aae1a997d5c406347 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T120405642786Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104565933056, + "active_bytes_peak_delta": 23630527488, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104565933056, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104565933056, + "device": 6, + "inactive_split_bytes_current": 5059658752, + "inactive_split_bytes_peak": 8053742592, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108227723264, + "reserved_bytes_peak": 108227723264, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-01T12:04:05.642372Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104565933056, + "allocated_bytes_peak_delta": 23630527488, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 80935405568, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 80935405568, + "device": 6, + "inactive_split_bytes_current": 5281956864, + "inactive_split_bytes_peak": 5390814720, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98075410432, + "reserved_bytes_peak": 98075410432, + "segment_count_current": 521, + "segment_count_peak": 521, + "timestamp_utc": "2026-08-01T12:04:04.801856Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5059658752, + "inactive_split_bytes_current_delta": -222298112, + "inactive_split_bytes_peak": 8053742592, + "inactive_split_bytes_peak_delta": 2662927872, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108227723264, + "reserved_bytes_current_delta": 10152312832, + "reserved_bytes_peak": 108227723264, + "reserved_bytes_peak_delta": 10152312832, + "segment_count_current": 558, + "segment_count_current_delta": 37, + "segment_count_peak": 558, + "segment_count_peak_delta": 37, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T120405642786Z.pickle", + "timestamp_utc": "2026-08-01T12:04:05.754434Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T120405642786Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T120405642786Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..71958adbb62e74b08c6c0c443481f60643e9d1dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T120405642786Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7071e95e729e441ddbf66db8fbf41f9ee0f2f739499b46f127203230c29576d9 +size 5590151 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T120406578079Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T120406578079Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9996391b227a5d449604c34d8e3ecd6d7c6638e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T120406578079Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870557696, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104570242560, + "active_bytes_peak_delta": 4309504, + "after": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104570242560, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104570242560, + "device": 6, + "inactive_split_bytes_current": 5063853056, + "inactive_split_bytes_peak": 8053742592, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108227723264, + "reserved_bytes_peak": 108227723264, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-01T12:04:06.577692Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870557696, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104570242560, + "allocated_bytes_peak_delta": 4309504, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104565933056, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104565933056, + "device": 6, + "inactive_split_bytes_current": 5059658752, + "inactive_split_bytes_peak": 8053742592, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108227723264, + "reserved_bytes_peak": 108227723264, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-01T12:04:05.772314Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5063853056, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8053742592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108227723264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108227723264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 558, + "segment_count_current_delta": 0, + "segment_count_peak": 558, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T120406578079Z.pickle", + "timestamp_utc": "2026-08-01T12:04:06.682847Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T120406578079Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T120406578079Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..05b13ff3bf5ee327bfdad10a54665f02a5635341 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T120406578079Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1a5d2c71ea516e44196c3852e4cfdc7a8917bbe4f830436c4caaa83ad2dbe72 +size 5582128 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T120409191445Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T120409191445Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2b2342e14c8af4b5fa7fb73e70e5a23f29ed2d79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T120409191445Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 156562808832, + "active_bytes_peak_delta": 51992566272, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_peak": 11760633856, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 170993385472, + "reserved_bytes_peak": 170993385472, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-01T12:04:09.190749Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104586182656, + "allocated_bytes_peak_delta": 15940096, + "before": { + "active_bytes_current": 66870557696, + "active_bytes_peak": 104570242560, + "allocated_bytes_current": 66870557696, + "allocated_bytes_peak": 104570242560, + "device": 6, + "inactive_split_bytes_current": 5063853056, + "inactive_split_bytes_peak": 8053742592, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108227723264, + "reserved_bytes_peak": 108227723264, + "segment_count_current": 558, + "segment_count_peak": 558, + "timestamp_utc": "2026-08-01T12:04:06.697623Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_current_delta": 196866048, + "inactive_split_bytes_peak": 11760633856, + "inactive_split_bytes_peak_delta": 3706891264, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 170993385472, + "reserved_bytes_current_delta": 62765662208, + "reserved_bytes_peak": 170993385472, + "reserved_bytes_peak_delta": 62765662208, + "segment_count_current": 929, + "segment_count_current_delta": 371, + "segment_count_peak": 929, + "segment_count_peak_delta": 371, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T120409191445Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.314311Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T120409191445Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T120409191445Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf5d483e5925e5411bc9ed36a5b349f0ef7bb601 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T120409191445Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ea92a987456ee29f12b08afb9b7afee05f20b1c97988a452faa9cb1898c03af +size 6593476 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T120409878001Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T120409878001Z.json new file mode 100644 index 0000000000000000000000000000000000000000..91af6e1fcfe79963bcdbddf732d1baf24be4edfa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T120409878001Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156562808832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_peak": 11760633856, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 170993385472, + "reserved_bytes_peak": 170993385472, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-01T12:04:09.877529Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586182656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_peak": 11760633856, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 170993385472, + "reserved_bytes_peak": 170993385472, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-01T12:04:09.331216Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11760633856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 170993385472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170993385472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 929, + "segment_count_current_delta": 0, + "segment_count_peak": 929, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T120409878001Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.885479Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T120409878001Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T120409878001Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e95560914d2ba55302a49622a228becb048d88d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T120409878001Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e7c7abd2b9f8c3f9eaac79367caf2cf7b90917dbcf4545dd77abb72bc1cdc2d +size 272030 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T120410434516Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T120410434516Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d5d7a0a46d901a0d60b226c3d3cb1808b18d680b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T120410434516Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156562808832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_peak": 11760633856, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 170993385472, + "reserved_bytes_peak": 170993385472, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-01T12:04:10.434088Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586182656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_peak": 11760633856, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 170993385472, + "reserved_bytes_peak": 170993385472, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-01T12:04:09.886463Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11760633856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 170993385472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170993385472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 929, + "segment_count_current_delta": 0, + "segment_count_peak": 929, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T120410434516Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.441467Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T120410434516Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T120410434516Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f6e1ec587a14205dda61b6b5c1d1f64bd59f2cad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T120410434516Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bf6d4d5c3d0731c1d81c9fb6c3d6ba6a6df41f1c7015b5cd83b7977987c1a45 +size 271999 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T120410988490Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T120410988490Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1dc9fedef3e91dc8474ca503632a2f98a4304d59 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T120410988490Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156562808832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_peak": 11760633856, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 170993385472, + "reserved_bytes_peak": 170993385472, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-01T12:04:10.988103Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586182656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_peak": 11760633856, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 170993385472, + "reserved_bytes_peak": 170993385472, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-01T12:04:10.442448Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11760633856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 170993385472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170993385472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 929, + "segment_count_current_delta": 0, + "segment_count_peak": 929, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T120410988490Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.994821Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T120410988490Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T120410988490Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba2d143ab9217c2fa701b0da89220412889df349 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T120410988490Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbeae23591c800081b9586362af3c030974c484cb3691dca82bdc8a02ac00bac +size 272030 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T120411542074Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T120411542074Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ecdd538be5c1a73cc2e11c4177980a73c341f62 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T120411542074Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156562808832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_peak": 11760633856, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 170993385472, + "reserved_bytes_peak": 170993385472, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-01T12:04:11.541680Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586182656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_peak": 11760633856, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 170993385472, + "reserved_bytes_peak": 170993385472, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-01T12:04:10.995782Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11760633856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 170993385472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170993385472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 929, + "segment_count_current_delta": 0, + "segment_count_peak": 929, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T120411542074Z.pickle", + "timestamp_utc": "2026-08-01T12:04:11.548452Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T120411542074Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T120411542074Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b096fb0edfefdc4ad9726c7286c063476f76fd18 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T120411542074Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d05aa08ad9adbed028defb08da1c17b7d58fb1e50ac052b74376f80448a6fb9d +size 271999 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T120412099215Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T120412099215Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d31c24a3ff61e379d96f3261aa16fb73a4553dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T120412099215Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156562808832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_peak": 11760633856, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 170993385472, + "reserved_bytes_peak": 170993385472, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-01T12:04:12.098814Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586182656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_peak": 11760633856, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 170993385472, + "reserved_bytes_peak": 170993385472, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-01T12:04:11.549417Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11760633856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 170993385472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 170993385472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 929, + "segment_count_current_delta": 0, + "segment_count_peak": 929, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T120412099215Z.pickle", + "timestamp_utc": "2026-08-01T12:04:12.105603Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T120412099215Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T120412099215Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d620a473088c173e0f9fa4d5779b5b6df86db09 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T120412099215Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f17313dc422b52a32094d119141f64b455a9a2e69219ba5db1095e04e1771673 +size 272030 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T120458246432Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T120458246432Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab30e4b28c0169a0f1bad8cc58997cb618d6c9cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T120458246432Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853966848, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 156562808832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853966848, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66853966848, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5004946432, + "inactive_split_bytes_peak": 11760633856, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 172192956416, + "reserved_bytes_peak": 172192956416, + "segment_count_current": 935, + "segment_count_peak": 935, + "timestamp_utc": "2026-08-01T12:04:58.245943Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853966848, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104586182656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601152, + "active_bytes_peak": 156562808832, + "allocated_bytes_current": 66887601152, + "allocated_bytes_peak": 104586182656, + "device": 6, + "inactive_split_bytes_current": 5260719104, + "inactive_split_bytes_peak": 11760633856, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 170993385472, + "reserved_bytes_peak": 170993385472, + "segment_count_current": 929, + "segment_count_peak": 929, + "timestamp_utc": "2026-08-01T12:04:12.106558Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5004946432, + "inactive_split_bytes_current_delta": -255772672, + "inactive_split_bytes_peak": 11760633856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 172192956416, + "reserved_bytes_current_delta": 1199570944, + "reserved_bytes_peak": 172192956416, + "reserved_bytes_peak_delta": 1199570944, + "segment_count_current": 935, + "segment_count_current_delta": 6, + "segment_count_peak": 935, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T120458246432Z.pickle", + "timestamp_utc": "2026-08-01T12:04:58.332586Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T120458246432Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T120458246432Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d8e842ee451a5816a3bc2f6fa2006986276bb28c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T120458246432Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71319032dca2fe98e45c3620bf5013db0289b03f5e1fb86aaaa9b61f43989e3a +size 3940284 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T120404665171Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T120404665171Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cedefd8dc307f8ab3e0eb963fd7913935c59529d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T120404665171Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870558208, + "active_bytes_current_delta": 31639102976, + "active_bytes_peak": 80939010048, + "active_bytes_peak_delta": 45472673792, + "after": { + "active_bytes_current": 66870558208, + "active_bytes_peak": 80939010048, + "allocated_bytes_current": 66870558208, + "allocated_bytes_peak": 80939010048, + "device": 7, + "inactive_split_bytes_current": 5401494016, + "inactive_split_bytes_peak": 5723764736, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 97106526208, + "reserved_bytes_peak": 97106526208, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:04:04.664697Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870558208, + "allocated_bytes_current_delta": 31639102976, + "allocated_bytes_peak": 80939010048, + "allocated_bytes_peak_delta": 45472673792, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:02:54.695723Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5401494016, + "inactive_split_bytes_current_delta": 5268675072, + "inactive_split_bytes_peak": 5723764736, + "inactive_split_bytes_peak_delta": 5563378688, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 97106526208, + "reserved_bytes_current_delta": 57862520832, + "reserved_bytes_peak": 97106526208, + "reserved_bytes_peak_delta": 57862520832, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T120404665171Z.pickle", + "timestamp_utc": "2026-08-01T12:04:04.780477Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T120404665171Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T120404665171Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c3c0800bc01b1e08343ca8eb1ddc30a9c747b1aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T120404665171Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7e6dd23852b3f4a72f011366674e3f3f0aae0b02ca4f4f73ca9182d3ff44b18 +size 5387386 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T120405642757Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T120405642757Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3942bab06ccd7ff95726625a0f44890f0117e8f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T120405642757Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870558208, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104577192448, + "active_bytes_peak_delta": 23638182400, + "after": { + "active_bytes_current": 66870558208, + "active_bytes_peak": 104577192448, + "allocated_bytes_current": 66870558208, + "allocated_bytes_peak": 104577192448, + "device": 7, + "inactive_split_bytes_current": 5653152256, + "inactive_split_bytes_peak": 8220858880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 109488111616, + "reserved_bytes_peak": 109488111616, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T12:04:05.642350Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870558208, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104577192448, + "allocated_bytes_peak_delta": 23638182400, + "before": { + "active_bytes_current": 66870558208, + "active_bytes_peak": 80939010048, + "allocated_bytes_current": 66870558208, + "allocated_bytes_peak": 80939010048, + "device": 7, + "inactive_split_bytes_current": 5401494016, + "inactive_split_bytes_peak": 5723764736, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 97106526208, + "reserved_bytes_peak": 97106526208, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:04:04.798168Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5653152256, + "inactive_split_bytes_current_delta": 251658240, + "inactive_split_bytes_peak": 8220858880, + "inactive_split_bytes_peak_delta": 2497094144, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 109488111616, + "reserved_bytes_current_delta": 12381585408, + "reserved_bytes_peak": 109488111616, + "reserved_bytes_peak_delta": 12381585408, + "segment_count_current": 565, + "segment_count_current_delta": 53, + "segment_count_peak": 565, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T120405642757Z.pickle", + "timestamp_utc": "2026-08-01T12:04:05.746523Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T120405642757Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T120405642757Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..81348509d48146953e5506399c3401ed61ba60df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T120405642757Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29ca2564a5d8a8d4540657ce183ab0719ef3d91ea8f8c48aabf5201ab3c45e6e +size 5592635 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T120406578067Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T120406578067Z.json new file mode 100644 index 0000000000000000000000000000000000000000..79187d93f96063a603f1610b409d8d7192f11c0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T120406578067Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66870558208, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104577192448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66870558208, + "active_bytes_peak": 104577192448, + "allocated_bytes_current": 66870558208, + "allocated_bytes_peak": 104577192448, + "device": 7, + "inactive_split_bytes_current": 5179195904, + "inactive_split_bytes_peak": 8220858880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 109488111616, + "reserved_bytes_peak": 109488111616, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T12:04:06.577676Z", + "world_size": 8 + }, + "allocated_bytes_current": 66870558208, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104577192448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66870558208, + "active_bytes_peak": 104577192448, + "allocated_bytes_current": 66870558208, + "allocated_bytes_peak": 104577192448, + "device": 7, + "inactive_split_bytes_current": 5653152256, + "inactive_split_bytes_peak": 8220858880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 109488111616, + "reserved_bytes_peak": 109488111616, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T12:04:05.759972Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5179195904, + "inactive_split_bytes_current_delta": -473956352, + "inactive_split_bytes_peak": 8220858880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 109488111616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109488111616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 565, + "segment_count_current_delta": 0, + "segment_count_peak": 565, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T120406578067Z.pickle", + "timestamp_utc": "2026-08-01T12:04:06.679075Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T120406578067Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T120406578067Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ab4a167f6d809fff4b5d864043128cf14ca7b00d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T120406578067Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9659363b65a1014641d21d9ae7cc60d61048e5ce87d11e1d86d6e4517829a6ae +size 5572568 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T120409191319Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T120409191319Z.json new file mode 100644 index 0000000000000000000000000000000000000000..082bf260169a6802a2a7aa06d7b9a035cca5a91b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T120409191319Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601664, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 156556452352, + "active_bytes_peak_delta": 51979259904, + "after": { + "active_bytes_current": 66887601664, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66887601664, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_peak": 12097478144, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.190697Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601664, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104586706432, + "allocated_bytes_peak_delta": 9513984, + "before": { + "active_bytes_current": 66870558208, + "active_bytes_peak": 104577192448, + "allocated_bytes_current": 66870558208, + "allocated_bytes_peak": 104577192448, + "device": 7, + "inactive_split_bytes_current": 5179195904, + "inactive_split_bytes_peak": 8220858880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 109488111616, + "reserved_bytes_peak": 109488111616, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T12:04:06.692054Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_current_delta": 201060352, + "inactive_split_bytes_peak": 12097478144, + "inactive_split_bytes_peak_delta": 3876619264, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 61620617216, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 61620617216, + "segment_count_current": 927, + "segment_count_current_delta": 362, + "segment_count_peak": 927, + "segment_count_peak_delta": 362, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T120409191319Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.313335Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T120409191319Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T120409191319Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3877e767bd0bd2f32947f31d9a96554470418e22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T120409191319Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:424cf543dafee53ac64a910f0c43add12ae74a2b7b39994245b892a32d580f79 +size 6597570 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T120409878017Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T120409878017Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04a5cf7e4bcca5514d18ceb4b35e2429ed8976e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T120409878017Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601664, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156556452352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601664, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66887601664, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_peak": 12097478144, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.877552Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601664, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586706432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601664, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66887601664, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_peak": 12097478144, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.329134Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12097478144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T120409878017Z.pickle", + "timestamp_utc": "2026-08-01T12:04:09.885208Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T120409878017Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T120409878017Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..20b09034bddb0dd91a8f6e0ffd618f7b78b86099 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T120409878017Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed09f53f0d59b4eb6c2bd17c71e4f1b9208c849d4495beb75fd1726af7716205 +size 273879 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T120410434503Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T120410434503Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f85451c09ba26b4a7fe358d9765b03bbe9439c3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T120410434503Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601664, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156556452352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601664, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66887601664, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_peak": 12097478144, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.434105Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601664, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586706432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601664, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66887601664, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_peak": 12097478144, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:09.886146Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12097478144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T120410434503Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.441122Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T120410434503Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T120410434503Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f8db3c3a2085fea1ea3dfd42704210bc801513d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T120410434503Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4801f9a3410fe8f597aad7c0a11798d4f9976f01753500b97e1626e4c177134 +size 273879 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T120410988522Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T120410988522Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c094591c403a031f9cf994b60e94d455e9ffba88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T120410988522Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601664, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156556452352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601664, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66887601664, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_peak": 12097478144, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.988125Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601664, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586706432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601664, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66887601664, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_peak": 12097478144, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.442049Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12097478144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T120410988522Z.pickle", + "timestamp_utc": "2026-08-01T12:04:10.994706Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T120410988522Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T120410988522Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..980c5a255ea8eec7544c0a39a4c594b9069999bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T120410988522Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c9dd1850886b8e82432afcb60024e6b8c87c0d08ec041e409112836a2fffcc7 +size 273879 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T120411542089Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T120411542089Z.json new file mode 100644 index 0000000000000000000000000000000000000000..960e2727776461b0f8326b832b4c5cc3e18f8fa3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T120411542089Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601664, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156556452352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601664, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66887601664, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_peak": 12097478144, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:11.541706Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601664, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586706432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601664, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66887601664, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_peak": 12097478144, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:10.995632Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12097478144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T120411542089Z.pickle", + "timestamp_utc": "2026-08-01T12:04:11.548300Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T120411542089Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T120411542089Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..afc24ef2bfefa7ac85da334554d6055bf4c336bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T120411542089Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17a3f82404f8af239e4fc11812b96958e5ed9487c548e93dff590018e2b8cf28 +size 273879 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T120412099218Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T120412099218Z.json new file mode 100644 index 0000000000000000000000000000000000000000..649195a5f4150e6d8c746660d063084cab95caa1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T120412099218Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66887601664, + "active_bytes_current_delta": 0, + "active_bytes_peak": 156556452352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66887601664, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66887601664, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_peak": 12097478144, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:12.098820Z", + "world_size": 8 + }, + "allocated_bytes_current": 66887601664, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104586706432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601664, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66887601664, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_peak": 12097478144, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:11.549239Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12097478144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 171108728832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171108728832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 927, + "segment_count_current_delta": 0, + "segment_count_peak": 927, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T120412099218Z.pickle", + "timestamp_utc": "2026-08-01T12:04:12.105346Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T120412099218Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T120412099218Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e3f884f4cc3ba9befefb013ada4b855c31b485d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T120412099218Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d53c9c643fb5c1f18d9965cdb2f08e6c5f4c6362a5b8264d54cd86c206c01c3 +size 273879 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T120458246412Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T120458246412Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c242866e88d4dc80f034a6db9b9a5e3f79c3748c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T120458246412Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853967360, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 156556452352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853967360, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66853967360, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5122386432, + "inactive_split_bytes_peak": 12097478144, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 173247823872, + "reserved_bytes_peak": 173247823872, + "segment_count_current": 937, + "segment_count_peak": 937, + "timestamp_utc": "2026-08-01T12:04:58.245955Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853967360, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 104586706432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66887601664, + "active_bytes_peak": 156556452352, + "allocated_bytes_current": 66887601664, + "allocated_bytes_peak": 104586706432, + "device": 7, + "inactive_split_bytes_current": 5380256256, + "inactive_split_bytes_peak": 12097478144, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 171108728832, + "reserved_bytes_peak": 171108728832, + "segment_count_current": 927, + "segment_count_peak": 927, + "timestamp_utc": "2026-08-01T12:04:12.106278Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5122386432, + "inactive_split_bytes_current_delta": -257869824, + "inactive_split_bytes_peak": 12097478144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 173247823872, + "reserved_bytes_current_delta": 2139095040, + "reserved_bytes_peak": 173247823872, + "reserved_bytes_peak_delta": 2139095040, + "segment_count_current": 937, + "segment_count_current_delta": 10, + "segment_count_peak": 937, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T120458246412Z.pickle", + "timestamp_utc": "2026-08-01T12:04:58.333813Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T120458246412Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T120458246412Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a913414bc2e9f876c8f8c1ecffdc643074ce7ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T120458246412Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcdeb63fbac9db88916a7027c3bc4967c5d9d1518cfe1d32a0fadcd5bb138720 +size 3943954 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..058ba6ba9091655d23b450f322593a07004242e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 2048 +capacity: 1.25 +expert_dispatch_overlap: on +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6007 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 230 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T120754712893Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T120754712893Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bffe9840c8f3764431c75200040ca4b2413449fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T120754712893Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459101184, + "active_bytes_current_delta": 37227645952, + "active_bytes_peak": 81183342080, + "active_bytes_peak_delta": 45717005824, + "after": { + "active_bytes_current": 72459101184, + "active_bytes_peak": 81183342080, + "allocated_bytes_current": 72459101184, + "allocated_bytes_peak": 80843061760, + "device": 0, + "inactive_split_bytes_current": 5215214592, + "inactive_split_bytes_peak": 9381037568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 97811169280, + "reserved_bytes_peak": 97811169280, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T12:07:54.712293Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459101184, + "allocated_bytes_current_delta": 37227645952, + "allocated_bytes_peak": 80843061760, + "allocated_bytes_peak_delta": 45376725504, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:06:45.209934Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5215214592, + "inactive_split_bytes_current_delta": 5080298496, + "inactive_split_bytes_peak": 9381037568, + "inactive_split_bytes_peak_delta": 9220405760, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 97811169280, + "reserved_bytes_current_delta": 58565066752, + "reserved_bytes_peak": 97811169280, + "reserved_bytes_peak_delta": 58565066752, + "segment_count_current": 466, + "segment_count_current_delta": 394, + "segment_count_peak": 466, + "segment_count_peak_delta": 394, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T120754712893Z.pickle", + "timestamp_utc": "2026-08-01T12:07:54.829683Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T120754712893Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T120754712893Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d8e685796291fbc829330b99022388663232c11b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T120754712893Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b24105608ab70415c1302a286c3ebac9721082062f41f3a0bb17055e578e167 +size 5950590 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T120755914308Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T120755914308Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a67f64159ccd57929d2061462b2ace87405b3c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T120755914308Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459101184, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104828837376, + "active_bytes_peak_delta": 23645495296, + "after": { + "active_bytes_current": 72459101184, + "active_bytes_peak": 104828837376, + "allocated_bytes_current": 72459101184, + "allocated_bytes_peak": 104481312768, + "device": 0, + "inactive_split_bytes_current": 6156835840, + "inactive_split_bytes_peak": 12215619584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109303562240, + "reserved_bytes_peak": 109303562240, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T12:07:55.913822Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459101184, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104481312768, + "allocated_bytes_peak_delta": 23638251008, + "before": { + "active_bytes_current": 72459101184, + "active_bytes_peak": 81183342080, + "allocated_bytes_current": 72459101184, + "allocated_bytes_peak": 80843061760, + "device": 0, + "inactive_split_bytes_current": 5215214592, + "inactive_split_bytes_peak": 9381037568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 97811169280, + "reserved_bytes_peak": 97811169280, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T12:07:54.849042Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6156835840, + "inactive_split_bytes_current_delta": 941621248, + "inactive_split_bytes_peak": 12215619584, + "inactive_split_bytes_peak_delta": 2834582016, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109303562240, + "reserved_bytes_current_delta": 11492392960, + "reserved_bytes_peak": 109303562240, + "reserved_bytes_peak_delta": 11492392960, + "segment_count_current": 519, + "segment_count_current_delta": 53, + "segment_count_peak": 519, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T120755914308Z.pickle", + "timestamp_utc": "2026-08-01T12:07:56.024365Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T120755914308Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T120755914308Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd8fc78b867444832b8b6e0e65f4cd6e51aadb3a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T120755914308Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9281a43bb9c1e3f11610b65d9b8ba9638dc416315e1d57cb47b3a8f62a102013 +size 6615521 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T120757013040Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T120757013040Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d7aa91aa9153bc40afe3388ddcad2c9cf77f58ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T120757013040Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459101184, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104828837376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72459101184, + "active_bytes_peak": 104828837376, + "allocated_bytes_current": 72459101184, + "allocated_bytes_peak": 104488316416, + "device": 0, + "inactive_split_bytes_current": 5926149120, + "inactive_split_bytes_peak": 12215619584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109303562240, + "reserved_bytes_peak": 109303562240, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T12:07:57.012571Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459101184, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104488316416, + "allocated_bytes_peak_delta": 7003648, + "before": { + "active_bytes_current": 72459101184, + "active_bytes_peak": 104828837376, + "allocated_bytes_current": 72459101184, + "allocated_bytes_peak": 104481312768, + "device": 0, + "inactive_split_bytes_current": 6156835840, + "inactive_split_bytes_peak": 12215619584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109303562240, + "reserved_bytes_peak": 109303562240, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T12:07:56.039266Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5926149120, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 12215619584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109303562240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109303562240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 519, + "segment_count_current_delta": 0, + "segment_count_peak": 519, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T120757013040Z.pickle", + "timestamp_utc": "2026-08-01T12:07:57.122361Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T120757013040Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T120757013040Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..46bb5abf03a42afa4f3671bfcf6be9c23689c794 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T120757013040Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f23939da1bad4504976ee4d70794e25ec82019a0c6e36c4e830d323c5edd5f8 +size 6595794 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T120759899649Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T120759899649Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad3c5d5457a287b13881cef33d6df5fed209aa75 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T120759899649Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72476144640, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162098030592, + "active_bytes_peak_delta": 57269193216, + "after": { + "active_bytes_current": 72476144640, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 72476144640, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_peak": 16229449728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_peak": 180816445440, + "segment_count_current": 862, + "segment_count_peak": 862, + "timestamp_utc": "2026-08-01T12:07:59.899076Z", + "world_size": 8 + }, + "allocated_bytes_current": 72476144640, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104501739520, + "allocated_bytes_peak_delta": 13423104, + "before": { + "active_bytes_current": 72459101184, + "active_bytes_peak": 104828837376, + "allocated_bytes_current": 72459101184, + "allocated_bytes_peak": 104488316416, + "device": 0, + "inactive_split_bytes_current": 5926149120, + "inactive_split_bytes_peak": 12215619584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109303562240, + "reserved_bytes_peak": 109303562240, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T12:07:57.136879Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_current_delta": 4288409600, + "inactive_split_bytes_peak": 16229449728, + "inactive_split_bytes_peak_delta": 4013830144, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_current_delta": 71512883200, + "reserved_bytes_peak": 180816445440, + "reserved_bytes_peak_delta": 71512883200, + "segment_count_current": 862, + "segment_count_current_delta": 343, + "segment_count_peak": 862, + "segment_count_peak_delta": 343, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T120759899649Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.023330Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T120759899649Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T120759899649Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0901c8b89d3e59454b395e46ce48759ac97c2c75 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T120759899649Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ef3c5bcf972b87c665ba0c6b66f8d453ad3fe67b2664b86a0f96e7af0c905af +size 6936330 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T120800855489Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T120800855489Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b55aa9171692f0194f664ac4d5b229e08a015479 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T120800855489Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72476144640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162098030592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72476144640, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 72476144640, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_peak": 16229449728, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_peak": 180816445440, + "segment_count_current": 862, + "segment_count_peak": 862, + "timestamp_utc": "2026-08-01T12:08:00.854977Z", + "world_size": 8 + }, + "allocated_bytes_current": 72476144640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104501739520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72476144640, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 72476144640, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_peak": 16229449728, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_peak": 180816445440, + "segment_count_current": 862, + "segment_count_peak": 862, + "timestamp_utc": "2026-08-01T12:08:00.040085Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16229449728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180816445440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 862, + "segment_count_current_delta": 0, + "segment_count_peak": 862, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T120800855489Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.861738Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T120800855489Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T120800855489Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..21e38f30fec1e3254ca566c198581abd7ef2f448 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T120800855489Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:480ac987fa28a52c42201bccb92d4063f945adb1422e53ccee60cc8244da20c0 +size 222323 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T120801671934Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T120801671934Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0e13984014fb17ae4d76f74a5d84a6d703c25dbc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T120801671934Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72476144640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162098030592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72476144640, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 72476144640, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_peak": 16229449728, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_peak": 180816445440, + "segment_count_current": 862, + "segment_count_peak": 862, + "timestamp_utc": "2026-08-01T12:08:01.671509Z", + "world_size": 8 + }, + "allocated_bytes_current": 72476144640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104501739520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72476144640, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 72476144640, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_peak": 16229449728, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_peak": 180816445440, + "segment_count_current": 862, + "segment_count_peak": 862, + "timestamp_utc": "2026-08-01T12:08:00.862700Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16229449728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180816445440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 862, + "segment_count_current_delta": 0, + "segment_count_peak": 862, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T120801671934Z.pickle", + "timestamp_utc": "2026-08-01T12:08:01.677730Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T120801671934Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T120801671934Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4d9ccf19eb61e0be427d04c716daf4046b48c24d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T120801671934Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cc453a3aee0a5806d503ddedf704952e841c3aa61df204472254d7409521424 +size 222329 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T120802490201Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T120802490201Z.json new file mode 100644 index 0000000000000000000000000000000000000000..24fcbfdc9cc10c3c06239fd515d1d261c57f518b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T120802490201Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72476144640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162098030592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72476144640, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 72476144640, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_peak": 16229449728, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_peak": 180816445440, + "segment_count_current": 862, + "segment_count_peak": 862, + "timestamp_utc": "2026-08-01T12:08:02.489789Z", + "world_size": 8 + }, + "allocated_bytes_current": 72476144640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104501739520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72476144640, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 72476144640, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_peak": 16229449728, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_peak": 180816445440, + "segment_count_current": 862, + "segment_count_peak": 862, + "timestamp_utc": "2026-08-01T12:08:01.678641Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16229449728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180816445440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 862, + "segment_count_current_delta": 0, + "segment_count_peak": 862, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T120802490201Z.pickle", + "timestamp_utc": "2026-08-01T12:08:02.495844Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T120802490201Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T120802490201Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..39ba1b6138d8716e121057b77c05fee83f75f62e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T120802490201Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45d128d61ed96ca609b8d6d398cef133567dc04becdd2b5b7854ff0af7c2fee1 +size 222323 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T120803305995Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T120803305995Z.json new file mode 100644 index 0000000000000000000000000000000000000000..affecebac6261bac4cf155685bdf03e3f4e08da4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T120803305995Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72476144640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162098030592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72476144640, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 72476144640, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_peak": 16229449728, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_peak": 180816445440, + "segment_count_current": 862, + "segment_count_peak": 862, + "timestamp_utc": "2026-08-01T12:08:03.305596Z", + "world_size": 8 + }, + "allocated_bytes_current": 72476144640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104501739520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72476144640, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 72476144640, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_peak": 16229449728, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_peak": 180816445440, + "segment_count_current": 862, + "segment_count_peak": 862, + "timestamp_utc": "2026-08-01T12:08:02.496762Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16229449728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180816445440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 862, + "segment_count_current_delta": 0, + "segment_count_peak": 862, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T120803305995Z.pickle", + "timestamp_utc": "2026-08-01T12:08:03.311492Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T120803305995Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T120803305995Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ddf9cc159813977695b76667f3292a6e0a586e14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T120803305995Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e93d83b07e7de712e7115c085ba68a618ed5c817c911810333f5f1376ca7c76 +size 222329 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T120804124961Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T120804124961Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3eca1a3d75016ad6ffa0db75b2ce4e623915fcf4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T120804124961Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72476144640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162098030592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72476144640, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 72476144640, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_peak": 16229449728, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_peak": 180816445440, + "segment_count_current": 862, + "segment_count_peak": 862, + "timestamp_utc": "2026-08-01T12:08:04.124568Z", + "world_size": 8 + }, + "allocated_bytes_current": 72476144640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104501739520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72476144640, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 72476144640, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_peak": 16229449728, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_peak": 180816445440, + "segment_count_current": 862, + "segment_count_peak": 862, + "timestamp_utc": "2026-08-01T12:08:03.312417Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16229449728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180816445440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 862, + "segment_count_current_delta": 0, + "segment_count_peak": 862, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T120804124961Z.pickle", + "timestamp_utc": "2026-08-01T12:08:04.130646Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T120804124961Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T120804124961Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4003d3bc2e36cee91eadadf1319b377c5d17f34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T120804124961Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97dca34a962d830484cdd4fedbeaf7ea7e4ae7419d03ee4ec244b6e6437af31a +size 222323 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T120851087341Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T120851087341Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ffa0b49765d1eb791d86421f93e0ce8ec1524f0d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T120851087341Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61218864128, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 162098030592, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61218864128, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 61218864128, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 6783386624, + "inactive_split_bytes_peak": 16229449728, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 1, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 171383455744, + "reserved_bytes_peak": 182743728128, + "segment_count_current": 819, + "segment_count_peak": 875, + "timestamp_utc": "2026-08-01T12:08:51.086859Z", + "world_size": 8 + }, + "allocated_bytes_current": 61218864128, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104501739520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72476144640, + "active_bytes_peak": 162098030592, + "allocated_bytes_current": 72476144640, + "allocated_bytes_peak": 104501739520, + "device": 0, + "inactive_split_bytes_current": 10214558720, + "inactive_split_bytes_peak": 16229449728, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 180816445440, + "reserved_bytes_peak": 180816445440, + "segment_count_current": 862, + "segment_count_peak": 862, + "timestamp_utc": "2026-08-01T12:08:04.131585Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6783386624, + "inactive_split_bytes_current_delta": -3431172096, + "inactive_split_bytes_peak": 16229449728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 1, + "num_alloc_retries_delta": 1, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 171383455744, + "reserved_bytes_current_delta": -9432989696, + "reserved_bytes_peak": 182743728128, + "reserved_bytes_peak_delta": 1927282688, + "segment_count_current": 819, + "segment_count_current_delta": -43, + "segment_count_peak": 875, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T120851087341Z.pickle", + "timestamp_utc": "2026-08-01T12:08:51.169873Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T120851087341Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T120851087341Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..20aa37ad0a0dcc9e3ecaaded4a659d6931be5592 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T120851087341Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e942122171fc1a844922bee9f0bd91fcdd6650df3e6374ba106bda6bd20940f +size 4164755 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T120754712799Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T120754712799Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ccc4c4a33b95d8147ea1b68a84705248bfae74ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T120754712799Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 81189726208, + "active_bytes_peak_delta": 45723389952, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 81189726208, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80850035712, + "device": 1, + "inactive_split_bytes_current": 5185799168, + "inactive_split_bytes_peak": 10743021056, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 98106867712, + "reserved_bytes_peak": 98106867712, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:07:54.712173Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 80850035712, + "allocated_bytes_peak_delta": 45383699456, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:06:45.210092Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5185799168, + "inactive_split_bytes_current_delta": 5052980224, + "inactive_split_bytes_peak": 10743021056, + "inactive_split_bytes_peak_delta": 10582635008, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 98106867712, + "reserved_bytes_current_delta": 58862862336, + "reserved_bytes_peak": 98106867712, + "reserved_bytes_peak_delta": 58862862336, + "segment_count_current": 468, + "segment_count_current_delta": 397, + "segment_count_peak": 468, + "segment_count_peak_delta": 397, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T120754712799Z.pickle", + "timestamp_utc": "2026-08-01T12:07:54.836017Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T120754712799Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T120754712799Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b85af785d1f598452c1d0f014fe16362eebb0f0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T120754712799Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f10e37a5536f0179c6d9d711afd8ec236a23ff94b837d3949145b8773706c5a +size 5942881 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T120755914344Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T120755914344Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f07342485c90b28c626e8ae1e2105df45e4ad891 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T120755914344Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104834137600, + "active_bytes_peak_delta": 23644411392, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104834137600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104482516992, + "device": 1, + "inactive_split_bytes_current": 5835916288, + "inactive_split_bytes_peak": 11768278016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 109551026176, + "reserved_bytes_peak": 109551026176, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T12:07:55.913859Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104482516992, + "allocated_bytes_peak_delta": 23632481280, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 81189726208, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80850035712, + "device": 1, + "inactive_split_bytes_current": 5185799168, + "inactive_split_bytes_peak": 10743021056, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 98106867712, + "reserved_bytes_peak": 98106867712, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:07:54.856276Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5835916288, + "inactive_split_bytes_current_delta": 650117120, + "inactive_split_bytes_peak": 11768278016, + "inactive_split_bytes_peak_delta": 1025256960, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 109551026176, + "reserved_bytes_current_delta": 11444158464, + "reserved_bytes_peak": 109551026176, + "reserved_bytes_peak_delta": 11444158464, + "segment_count_current": 519, + "segment_count_current_delta": 51, + "segment_count_peak": 519, + "segment_count_peak_delta": 51, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T120755914344Z.pickle", + "timestamp_utc": "2026-08-01T12:07:56.031494Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T120755914344Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T120755914344Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e63a67b32c13c6dc2727d6f9bccac1b3cedb109 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T120755914344Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33ef8ebdce0da983eb7b6e54d7acbfc3ee216080120f7bdccfebb12d61908ed6 +size 6049886 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T120757013083Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T120757013083Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4584c14ac8b9cb285048c958e4f89d09044981e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T120757013083Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104834137600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104834137600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104482516992, + "device": 1, + "inactive_split_bytes_current": 6041437184, + "inactive_split_bytes_peak": 11846023168, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 109551026176, + "reserved_bytes_peak": 109551026176, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T12:07:57.012609Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104482516992, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104834137600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104482516992, + "device": 1, + "inactive_split_bytes_current": 5835916288, + "inactive_split_bytes_peak": 11768278016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 109551026176, + "reserved_bytes_peak": 109551026176, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T12:07:56.048536Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6041437184, + "inactive_split_bytes_current_delta": 205520896, + "inactive_split_bytes_peak": 11846023168, + "inactive_split_bytes_peak_delta": 77745152, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 109551026176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109551026176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 519, + "segment_count_current_delta": 0, + "segment_count_peak": 519, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T120757013083Z.pickle", + "timestamp_utc": "2026-08-01T12:07:57.128011Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T120757013083Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T120757013083Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4afeb491bac77d240f12299d86e94034f56b5239 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T120757013083Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf1db5d7afd07f4213d7853eaae691b1d34821667754ff437be9ca67413e1585 +size 6594012 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T120759899634Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T120759899634Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bb82cf704f001530058ecbc244a1b683c0bc9faf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T120759899634Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162093629440, + "active_bytes_peak_delta": 57259491840, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_peak": 16268178432, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 181147795456, + "reserved_bytes_peak": 181147795456, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:07:59.899088Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104500419072, + "allocated_bytes_peak_delta": 17902080, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104834137600, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104482516992, + "device": 1, + "inactive_split_bytes_current": 6041437184, + "inactive_split_bytes_peak": 11846023168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 109551026176, + "reserved_bytes_peak": 109551026176, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T12:07:57.144574Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_current_delta": 3789287424, + "inactive_split_bytes_peak": 16268178432, + "inactive_split_bytes_peak_delta": 4422155264, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 181147795456, + "reserved_bytes_current_delta": 71596769280, + "reserved_bytes_peak": 181147795456, + "reserved_bytes_peak_delta": 71596769280, + "segment_count_current": 867, + "segment_count_current_delta": 348, + "segment_count_peak": 867, + "segment_count_peak_delta": 348, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T120759899634Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.028674Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T120759899634Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T120759899634Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..53f0fdcf62be91d2cd0c6a900491a36a6727fec5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T120759899634Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aef87f363a7b286eb7fb0b485108cc11622c398d3e5212740a503f2ec361aa41 +size 6936537 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T120800855468Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T120800855468Z.json new file mode 100644 index 0000000000000000000000000000000000000000..73e6bcddd699bb173b58e04ae4affab1db631213 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T120800855468Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162093629440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_peak": 16268178432, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 181147795456, + "reserved_bytes_peak": 181147795456, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:00.854973Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104500419072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_peak": 16268178432, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 181147795456, + "reserved_bytes_peak": 181147795456, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:00.047189Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16268178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 181147795456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181147795456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 867, + "segment_count_current_delta": 0, + "segment_count_peak": 867, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T120800855468Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.861920Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T120800855468Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T120800855468Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b7a7229e1f5c3fde784aa1e3cd769ec17b532dd7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T120800855468Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a14957b2028db13e2df5d43ca6f1689729da85bcfc95a6de4790eb0d5982d77 +size 223622 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T120801671903Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T120801671903Z.json new file mode 100644 index 0000000000000000000000000000000000000000..431ee6aee747242292b1a662997e26c3255975e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T120801671903Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162093629440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_peak": 16268178432, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 181147795456, + "reserved_bytes_peak": 181147795456, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:01.671487Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104500419072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_peak": 16268178432, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 181147795456, + "reserved_bytes_peak": 181147795456, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:00.862881Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16268178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 181147795456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181147795456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 867, + "segment_count_current_delta": 0, + "segment_count_peak": 867, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T120801671903Z.pickle", + "timestamp_utc": "2026-08-01T12:08:01.677698Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T120801671903Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T120801671903Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7ebc637113a3efb4d7cc9e73815b0d3d91c5227 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T120801671903Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae681427ada565628b79aade2c3cd4c40b3be212aacdd377e625206cbfed0bbe +size 223653 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T120802490150Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T120802490150Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd478622fe6e6be3e635dde4cf2cffd57ae633bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T120802490150Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162093629440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_peak": 16268178432, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 181147795456, + "reserved_bytes_peak": 181147795456, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:02.489760Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104500419072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_peak": 16268178432, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 181147795456, + "reserved_bytes_peak": 181147795456, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:01.678608Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16268178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 181147795456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181147795456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 867, + "segment_count_current_delta": 0, + "segment_count_peak": 867, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T120802490150Z.pickle", + "timestamp_utc": "2026-08-01T12:08:02.495778Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T120802490150Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T120802490150Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2c3c71f4e31035d8e6b095ff625dc5828d35ba34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T120802490150Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:950dbb5952f14c0c7a2c4c01d4548f4f2371006e4d525ad0d09b9d02e43aa9ab +size 223622 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T120803306034Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T120803306034Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d9897f4e4be248294e8bb800f284f91b726f31d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T120803306034Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162093629440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_peak": 16268178432, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 181147795456, + "reserved_bytes_peak": 181147795456, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:03.305594Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104500419072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_peak": 16268178432, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 181147795456, + "reserved_bytes_peak": 181147795456, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:02.496669Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16268178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 181147795456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181147795456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 867, + "segment_count_current_delta": 0, + "segment_count_peak": 867, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T120803306034Z.pickle", + "timestamp_utc": "2026-08-01T12:08:03.311536Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T120803306034Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T120803306034Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..746c796a4cbcab7abfe15d727c5717a4f535022e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T120803306034Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48557f05ca9688634db17671489e5edb87ac3667d33c0487d9c8fcb3357648b9 +size 223653 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T120804124926Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T120804124926Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f94e20bb106e3e310077198e642bf10e4fbbc9f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T120804124926Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162093629440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_peak": 16268178432, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 181147795456, + "reserved_bytes_peak": 181147795456, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:04.124517Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104500419072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_peak": 16268178432, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 181147795456, + "reserved_bytes_peak": 181147795456, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:03.312482Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16268178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 181147795456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181147795456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 867, + "segment_count_current_delta": 0, + "segment_count_peak": 867, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T120804124926Z.pickle", + "timestamp_utc": "2026-08-01T12:08:04.130432Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T120804124926Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T120804124926Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e9663f99e3f56b9a972a73cac25ca83a1232a289 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T120804124926Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a4de759e1de233d356f4f98c9b29e43dd9e6793d0a290c451ffc8ffc93884c3 +size 223622 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T120851087353Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T120851087353Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d91fc9bacc61f151fb744e50e47a6a23ae24c0ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T120851087353Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 162093629440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 6399552512, + "inactive_split_bytes_peak": 16268178432, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 1, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 172880822272, + "reserved_bytes_peak": 182840197120, + "segment_count_current": 828, + "segment_count_peak": 879, + "timestamp_utc": "2026-08-01T12:08:51.086854Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104500419072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162093629440, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104500419072, + "device": 1, + "inactive_split_bytes_current": 9830724608, + "inactive_split_bytes_peak": 16268178432, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 181147795456, + "reserved_bytes_peak": 181147795456, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:04.131354Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6399552512, + "inactive_split_bytes_current_delta": -3431172096, + "inactive_split_bytes_peak": 16268178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 1, + "num_alloc_retries_delta": 1, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 172880822272, + "reserved_bytes_current_delta": -8266973184, + "reserved_bytes_peak": 182840197120, + "reserved_bytes_peak_delta": 1692401664, + "segment_count_current": 828, + "segment_count_current_delta": -39, + "segment_count_peak": 879, + "segment_count_peak_delta": 12, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T120851087353Z.pickle", + "timestamp_utc": "2026-08-01T12:08:51.177027Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T120851087353Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T120851087353Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e8550e3463a208808e927aece5fb1ad8f7a3f59f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T120851087353Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:409e01165880425478fddcfece5519dcacd4f9be5c4bdb3cc478c7d79f516691 +size 4040635 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T120754712778Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T120754712778Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c44f16e00296ade6d271e90a2e3c997fb735686f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T120754712778Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 81192067072, + "active_bytes_peak_delta": 45725730816, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 81192067072, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80848987136, + "device": 2, + "inactive_split_bytes_current": 5097718784, + "inactive_split_bytes_peak": 8627339776, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 97456750592, + "reserved_bytes_peak": 97456750592, + "segment_count_current": 465, + "segment_count_peak": 465, + "timestamp_utc": "2026-08-01T12:07:54.712171Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 80848987136, + "allocated_bytes_peak_delta": 45382650880, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:06:45.210436Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5097718784, + "inactive_split_bytes_current_delta": 4964899840, + "inactive_split_bytes_peak": 8627339776, + "inactive_split_bytes_peak_delta": 8466953728, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 97456750592, + "reserved_bytes_current_delta": 58212745216, + "reserved_bytes_peak": 97456750592, + "reserved_bytes_peak_delta": 58212745216, + "segment_count_current": 465, + "segment_count_current_delta": 394, + "segment_count_peak": 465, + "segment_count_peak_delta": 394, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T120754712778Z.pickle", + "timestamp_utc": "2026-08-01T12:07:54.834828Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T120754712778Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T120754712778Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..33d42ee535fe83b0e6cec4694177f1668eb223b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T120754712778Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b00f0c13190315354979fd4bdbc6cfddd5846934cc74ea6956bb1b2eb9b667e +size 5942258 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T120755914204Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T120755914204Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7da4fe596d6de9de5a631d9ced3cd0f64c499eb3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T120755914204Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104833678848, + "active_bytes_peak_delta": 23641611776, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104833678848, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104482122752, + "device": 2, + "inactive_split_bytes_current": 6242763776, + "inactive_split_bytes_peak": 12104236032, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109385351168, + "reserved_bytes_peak": 109385351168, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:07:55.913782Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104482122752, + "allocated_bytes_peak_delta": 23633135616, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 81192067072, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80848987136, + "device": 2, + "inactive_split_bytes_current": 5097718784, + "inactive_split_bytes_peak": 8627339776, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 97456750592, + "reserved_bytes_peak": 97456750592, + "segment_count_current": 465, + "segment_count_peak": 465, + "timestamp_utc": "2026-08-01T12:07:54.851021Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6242763776, + "inactive_split_bytes_current_delta": 1145044992, + "inactive_split_bytes_peak": 12104236032, + "inactive_split_bytes_peak_delta": 3476896256, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109385351168, + "reserved_bytes_current_delta": 11928600576, + "reserved_bytes_peak": 109385351168, + "reserved_bytes_peak_delta": 11928600576, + "segment_count_current": 523, + "segment_count_current_delta": 58, + "segment_count_peak": 523, + "segment_count_peak_delta": 58, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T120755914204Z.pickle", + "timestamp_utc": "2026-08-01T12:07:56.024768Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T120755914204Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T120755914204Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6bf4cc19fa956989fb315f5d8d84c3407e1abc93 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T120755914204Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a50a2113e127aab9d8464959ca3ae59a8ea1390a1de43c507649809fe4b9209b +size 6056970 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T120757012891Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T120757012891Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d9f87ac27d8bd81afd7c99a4d18a68592011a318 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T120757012891Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104833678848, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104833678848, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104485395968, + "device": 2, + "inactive_split_bytes_current": 6337135616, + "inactive_split_bytes_peak": 12104236032, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109385351168, + "reserved_bytes_peak": 109385351168, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:07:57.012498Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104485395968, + "allocated_bytes_peak_delta": 3273216, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104833678848, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104482122752, + "device": 2, + "inactive_split_bytes_current": 6242763776, + "inactive_split_bytes_peak": 12104236032, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109385351168, + "reserved_bytes_peak": 109385351168, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:07:56.037557Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6337135616, + "inactive_split_bytes_current_delta": 94371840, + "inactive_split_bytes_peak": 12104236032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109385351168, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109385351168, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T120757012891Z.pickle", + "timestamp_utc": "2026-08-01T12:07:57.117439Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T120757012891Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T120757012891Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf6cab4304c2eb7ffb7fea3cfbdb4621f8db970e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T120757012891Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7ad111c2e4cff6627eb366fd8d80c97d9f0abde822ee2f437cfd7beb5797154 +size 6317931 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T120759899622Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T120759899622Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c0c995030ef92188b872e645300b3fb208e961e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T120759899622Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162099200000, + "active_bytes_peak_delta": 57265521152, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_peak": 16087823360, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 181443493888, + "reserved_bytes_peak": 181443493888, + "segment_count_current": 869, + "segment_count_peak": 869, + "timestamp_utc": "2026-08-01T12:07:59.899043Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104496421888, + "allocated_bytes_peak_delta": 11025920, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104833678848, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104485395968, + "device": 2, + "inactive_split_bytes_current": 6337135616, + "inactive_split_bytes_peak": 12104236032, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109385351168, + "reserved_bytes_peak": 109385351168, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:07:57.129924Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_current_delta": 2994466816, + "inactive_split_bytes_peak": 16087823360, + "inactive_split_bytes_peak_delta": 3983587328, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 181443493888, + "reserved_bytes_current_delta": 72058142720, + "reserved_bytes_peak": 181443493888, + "reserved_bytes_peak_delta": 72058142720, + "segment_count_current": 869, + "segment_count_current_delta": 346, + "segment_count_peak": 869, + "segment_count_peak_delta": 346, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T120759899622Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.019651Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T120759899622Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T120759899622Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5e31d3f5bef871a8eb4595d01b9bd26279a56c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T120759899622Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83be01135c815aba138b0e4176926588bdc026a61344be94fcdc4685e45463c0 +size 6936761 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T120800855394Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T120800855394Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d84465dc2d0293ab27efa4e7830a0c27e344585 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T120800855394Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162099200000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_peak": 16087823360, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 181443493888, + "reserved_bytes_peak": 181443493888, + "segment_count_current": 869, + "segment_count_peak": 869, + "timestamp_utc": "2026-08-01T12:08:00.854945Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496421888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_peak": 16087823360, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 181443493888, + "reserved_bytes_peak": 181443493888, + "segment_count_current": 869, + "segment_count_peak": 869, + "timestamp_utc": "2026-08-01T12:08:00.033174Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16087823360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 181443493888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181443493888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 869, + "segment_count_current_delta": 0, + "segment_count_peak": 869, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T120800855394Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.861001Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T120800855394Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T120800855394Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6a991f336bd823af1f575f134c32d6d0f42f18a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T120800855394Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7013c7331ffb43913471e009b2d984c6496d1298abc9e49a633b5e705701d189 +size 223929 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T120801671848Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T120801671848Z.json new file mode 100644 index 0000000000000000000000000000000000000000..946e9bd8e898038fa4c3787ae51f0e605dbe8aa6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T120801671848Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162099200000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_peak": 16087823360, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 181443493888, + "reserved_bytes_peak": 181443493888, + "segment_count_current": 869, + "segment_count_peak": 869, + "timestamp_utc": "2026-08-01T12:08:01.671458Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496421888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_peak": 16087823360, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 181443493888, + "reserved_bytes_peak": 181443493888, + "segment_count_current": 869, + "segment_count_peak": 869, + "timestamp_utc": "2026-08-01T12:08:00.861914Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16087823360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 181443493888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181443493888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 869, + "segment_count_current_delta": 0, + "segment_count_peak": 869, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T120801671848Z.pickle", + "timestamp_utc": "2026-08-01T12:08:01.677253Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T120801671848Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T120801671848Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2622bac18b987bfbd8275ab52a07d976aaf2405e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T120801671848Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa89e2d7188581e9edf0aab39943f11a91ad04ae375d43f3247b39d3d3132cb0 +size 223960 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T120802490136Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T120802490136Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aba400be050286c431ef9e0d7bb3fb0f84e731b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T120802490136Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162099200000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_peak": 16087823360, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 181443493888, + "reserved_bytes_peak": 181443493888, + "segment_count_current": 869, + "segment_count_peak": 869, + "timestamp_utc": "2026-08-01T12:08:02.489747Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496421888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_peak": 16087823360, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 181443493888, + "reserved_bytes_peak": 181443493888, + "segment_count_current": 869, + "segment_count_peak": 869, + "timestamp_utc": "2026-08-01T12:08:01.678145Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16087823360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 181443493888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181443493888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 869, + "segment_count_current_delta": 0, + "segment_count_peak": 869, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T120802490136Z.pickle", + "timestamp_utc": "2026-08-01T12:08:02.495541Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T120802490136Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T120802490136Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b5e02ce9de66f62498e65dc91f090985fed89e00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T120802490136Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6334c778e4726dc74ed951b2abb28fa8048bbc99105f6c71dc2048c77ce53fd +size 223929 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T120803305990Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T120803305990Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c124722fd8d8c5ee616960abdcaa5897bed54443 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T120803305990Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162099200000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_peak": 16087823360, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 181443493888, + "reserved_bytes_peak": 181443493888, + "segment_count_current": 869, + "segment_count_peak": 869, + "timestamp_utc": "2026-08-01T12:08:03.305593Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496421888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_peak": 16087823360, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 181443493888, + "reserved_bytes_peak": 181443493888, + "segment_count_current": 869, + "segment_count_peak": 869, + "timestamp_utc": "2026-08-01T12:08:02.496431Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16087823360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 181443493888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181443493888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 869, + "segment_count_current_delta": 0, + "segment_count_peak": 869, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T120803305990Z.pickle", + "timestamp_utc": "2026-08-01T12:08:03.311351Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T120803305990Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T120803305990Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..196503a70b81fb8d34e38cbd7f5d8db1c9fd1eec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T120803305990Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b11975a1f17f42c4271ca109c74b7556a7c465672208c89d41905358978c8d0a +size 223960 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T120804124905Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T120804124905Z.json new file mode 100644 index 0000000000000000000000000000000000000000..45001c332a09a618c1c850ef87b133cd4f320968 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T120804124905Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162099200000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_peak": 16087823360, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 181443493888, + "reserved_bytes_peak": 181443493888, + "segment_count_current": 869, + "segment_count_peak": 869, + "timestamp_utc": "2026-08-01T12:08:04.124520Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496421888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_peak": 16087823360, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 181443493888, + "reserved_bytes_peak": 181443493888, + "segment_count_current": 869, + "segment_count_peak": 869, + "timestamp_utc": "2026-08-01T12:08:03.312267Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16087823360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 181443493888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181443493888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 869, + "segment_count_current_delta": 0, + "segment_count_peak": 869, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T120804124905Z.pickle", + "timestamp_utc": "2026-08-01T12:08:04.130138Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T120804124905Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T120804124905Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1fca60f5b9450c9ea300a63ba8c095205cf86bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T120804124905Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4384cf71d0235ab5d3e4af812c43e8b0af4704e6c641158d247851e60d90796c +size 223929 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T120851087318Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T120851087318Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e4c60ac091561f5f4afb2b6e1e9c327c47c63ce0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T120851087318Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 162099200000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 5841710080, + "inactive_split_bytes_peak": 16087823360, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 1, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 172302008320, + "reserved_bytes_peak": 182901014528, + "segment_count_current": 826, + "segment_count_peak": 880, + "timestamp_utc": "2026-08-01T12:08:51.086837Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104496421888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099200000, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496421888, + "device": 2, + "inactive_split_bytes_current": 9331602432, + "inactive_split_bytes_peak": 16087823360, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 181443493888, + "reserved_bytes_peak": 181443493888, + "segment_count_current": 869, + "segment_count_peak": 869, + "timestamp_utc": "2026-08-01T12:08:04.131033Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5841710080, + "inactive_split_bytes_current_delta": -3489892352, + "inactive_split_bytes_peak": 16087823360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 1, + "num_alloc_retries_delta": 1, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 172302008320, + "reserved_bytes_current_delta": -9141485568, + "reserved_bytes_peak": 182901014528, + "reserved_bytes_peak_delta": 1457520640, + "segment_count_current": 826, + "segment_count_current_delta": -43, + "segment_count_peak": 880, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T120851087318Z.pickle", + "timestamp_utc": "2026-08-01T12:08:51.170526Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T120851087318Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T120851087318Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc11e67e2df7738b86a7d4d92c587dfd74888e88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T120851087318Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fadebb87cedf725ac2a427c81da1d5b4ee1db5251f510e5d3f270cfbfcd1ee4 +size 4040108 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T120754712859Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T120754712859Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4173219955c84b736b3e60b361350fe729ec1f41 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T120754712859Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 81537531392, + "active_bytes_peak_delta": 46071195136, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 81537531392, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80845775360, + "device": 3, + "inactive_split_bytes_current": 5687018496, + "inactive_split_bytes_peak": 9274637824, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 98016690176, + "reserved_bytes_peak": 98016690176, + "segment_count_current": 464, + "segment_count_peak": 464, + "timestamp_utc": "2026-08-01T12:07:54.712245Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 80845775360, + "allocated_bytes_peak_delta": 45379439104, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:06:45.210050Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5687018496, + "inactive_split_bytes_current_delta": 5554199552, + "inactive_split_bytes_peak": 9274637824, + "inactive_split_bytes_peak_delta": 9114251776, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 98016690176, + "reserved_bytes_current_delta": 58772684800, + "reserved_bytes_peak": 98016690176, + "reserved_bytes_peak_delta": 58772684800, + "segment_count_current": 464, + "segment_count_current_delta": 393, + "segment_count_peak": 464, + "segment_count_peak_delta": 393, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T120754712859Z.pickle", + "timestamp_utc": "2026-08-01T12:07:54.833387Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T120754712859Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T120754712859Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b3fe0f2ac0fbf4e30ea53e8178a4ced34fac02af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T120754712859Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:facc74bed99cb6fc70ffdbd86240924998951273bae8c00924365fd5cd8de5d1 +size 5942382 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T120755914224Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T120755914224Z.json new file mode 100644 index 0000000000000000000000000000000000000000..116da2593b79f163714798e83c7ac8d83ef68908 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T120755914224Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104843772416, + "active_bytes_peak_delta": 23306241024, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104843772416, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104491450880, + "device": 3, + "inactive_split_bytes_current": 6100157440, + "inactive_split_bytes_peak": 12126017536, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 109884473344, + "reserved_bytes_peak": 109884473344, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:07:55.913814Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104491450880, + "allocated_bytes_peak_delta": 23645675520, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 81537531392, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80845775360, + "device": 3, + "inactive_split_bytes_current": 5687018496, + "inactive_split_bytes_peak": 9274637824, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 98016690176, + "reserved_bytes_peak": 98016690176, + "segment_count_current": 464, + "segment_count_peak": 464, + "timestamp_utc": "2026-08-01T12:07:54.852460Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6100157440, + "inactive_split_bytes_current_delta": 413138944, + "inactive_split_bytes_peak": 12126017536, + "inactive_split_bytes_peak_delta": 2851379712, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 109884473344, + "reserved_bytes_current_delta": 11867783168, + "reserved_bytes_peak": 109884473344, + "reserved_bytes_peak_delta": 11867783168, + "segment_count_current": 523, + "segment_count_current_delta": 59, + "segment_count_peak": 523, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T120755914224Z.pickle", + "timestamp_utc": "2026-08-01T12:07:56.026333Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T120755914224Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T120755914224Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..705f35cf76ca6c428c50cf3def06ad71527ad411 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T120755914224Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:996f396ef5918f1e81619f8dc1ba26fb0aa82353c5c348ee8a128c4b58239275 +size 6582589 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T120757012888Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T120757012888Z.json new file mode 100644 index 0000000000000000000000000000000000000000..668452d0b3c55821436f3caa67f8c23fc00c5aca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T120757012888Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104843772416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104843772416, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104491450880, + "device": 3, + "inactive_split_bytes_current": 6450381824, + "inactive_split_bytes_peak": 12126017536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 109884473344, + "reserved_bytes_peak": 109884473344, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:07:57.012505Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104491450880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104843772416, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104491450880, + "device": 3, + "inactive_split_bytes_current": 6100157440, + "inactive_split_bytes_peak": 12126017536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 109884473344, + "reserved_bytes_peak": 109884473344, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:07:56.042409Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6450381824, + "inactive_split_bytes_current_delta": 350224384, + "inactive_split_bytes_peak": 12126017536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 109884473344, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109884473344, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T120757012888Z.pickle", + "timestamp_utc": "2026-08-01T12:07:57.122930Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T120757012888Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T120757012888Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..296124ddb4692e2b7c6a046f65d2750419e55ec3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T120757012888Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b9df6d4385dd056c288971956f6b5ed2bd040530abdfa1191b6e9a2fb64d885 +size 6596399 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T120759899814Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T120759899814Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ec85ed6d161ea3928f3743d920927287654cc830 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T120759899814Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162098741248, + "active_bytes_peak_delta": 57254968832, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_peak": 16082528256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 181481242624, + "reserved_bytes_peak": 181481242624, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:07:59.899180Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104496814080, + "allocated_bytes_peak_delta": 5363200, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104843772416, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104491450880, + "device": 3, + "inactive_split_bytes_current": 6450381824, + "inactive_split_bytes_peak": 12126017536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 109884473344, + "reserved_bytes_peak": 109884473344, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:07:57.138605Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_current_delta": 3766218752, + "inactive_split_bytes_peak": 16082528256, + "inactive_split_bytes_peak_delta": 3956510720, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 181481242624, + "reserved_bytes_current_delta": 71596769280, + "reserved_bytes_peak": 181481242624, + "reserved_bytes_peak_delta": 71596769280, + "segment_count_current": 867, + "segment_count_current_delta": 344, + "segment_count_peak": 867, + "segment_count_peak_delta": 344, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T120759899814Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.027966Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T120759899814Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T120759899814Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6966b30428731c5c1647733424685bacbff2700 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T120759899814Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75ee3659541ab5f936471bb4d7a91d761db2be560fd178a3f2c0f0a6866943b8 +size 6938538 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T120800855459Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T120800855459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96be642cc7c53e3d8a36b3eb9a38ec1e90d6eade --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T120800855459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162098741248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_peak": 16082528256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 181481242624, + "reserved_bytes_peak": 181481242624, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:00.854973Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496814080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_peak": 16082528256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 181481242624, + "reserved_bytes_peak": 181481242624, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:00.046223Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16082528256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 181481242624, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181481242624, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 867, + "segment_count_current_delta": 0, + "segment_count_peak": 867, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T120800855459Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.861826Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T120800855459Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T120800855459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c9fde22bcf0e72c6e6a61d988ed3be2e1841bb5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T120800855459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38bc332c32d88fd3b3c13c2179ac7b9a7a44e03e7f7250c2e03eb8bee25ace09 +size 223506 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T120801671928Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T120801671928Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1bd33b29df7a9de5eaf96a03ef4ba935a28cff1d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T120801671928Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162098741248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_peak": 16082528256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 181481242624, + "reserved_bytes_peak": 181481242624, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:01.671519Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496814080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_peak": 16082528256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 181481242624, + "reserved_bytes_peak": 181481242624, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:00.862789Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16082528256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 181481242624, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181481242624, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 867, + "segment_count_current_delta": 0, + "segment_count_peak": 867, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T120801671928Z.pickle", + "timestamp_utc": "2026-08-01T12:08:01.677635Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T120801671928Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T120801671928Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2722ec93b8f3cfd425465848a3ea0397bee69853 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T120801671928Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:967a7c53e203d40adf57da4725675d1db0a8d703762fa81687f3991a1bb72944 +size 223506 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T120802490212Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T120802490212Z.json new file mode 100644 index 0000000000000000000000000000000000000000..89b6223bf44b4183ab1194cb8059ba813239d63e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T120802490212Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162098741248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_peak": 16082528256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 181481242624, + "reserved_bytes_peak": 181481242624, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:02.489806Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496814080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_peak": 16082528256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 181481242624, + "reserved_bytes_peak": 181481242624, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:01.678543Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16082528256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 181481242624, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181481242624, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 867, + "segment_count_current_delta": 0, + "segment_count_peak": 867, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T120802490212Z.pickle", + "timestamp_utc": "2026-08-01T12:08:02.496082Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T120802490212Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T120802490212Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b910b41621840875bf04562a8adea79aa4ff5a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T120802490212Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:317c8e78ae125d0078d370157c87bd3b71b855f1d5dd3cde5b2ce76c12eb9a55 +size 223506 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T120803305992Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T120803305992Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7eafdad2b2247c45b804bf8304bf27745d2e5daf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T120803305992Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162098741248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_peak": 16082528256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 181481242624, + "reserved_bytes_peak": 181481242624, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:03.305599Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496814080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_peak": 16082528256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 181481242624, + "reserved_bytes_peak": 181481242624, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:02.496983Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16082528256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 181481242624, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181481242624, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 867, + "segment_count_current_delta": 0, + "segment_count_peak": 867, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T120803305992Z.pickle", + "timestamp_utc": "2026-08-01T12:08:03.311810Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T120803305992Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T120803305992Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e4c3346498efd71df74ce5fd1b9ebdcb9c8fdb47 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T120803305992Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2048ede03aebb7b22d7a56652a98454a78ce0371e01bdf3bf55e11e217d49940 +size 223506 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T120804125014Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T120804125014Z.json new file mode 100644 index 0000000000000000000000000000000000000000..676509f1b396e334d6187a3bc4325fdd55220d95 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T120804125014Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162098741248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_peak": 16082528256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 181481242624, + "reserved_bytes_peak": 181481242624, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:04.124607Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496814080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_peak": 16082528256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 181481242624, + "reserved_bytes_peak": 181481242624, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:03.312733Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16082528256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 181481242624, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181481242624, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 867, + "segment_count_current_delta": 0, + "segment_count_peak": 867, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T120804125014Z.pickle", + "timestamp_utc": "2026-08-01T12:08:04.131315Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T120804125014Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T120804125014Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..62396c4e236fc6c92bd259b8872431264f3818b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T120804125014Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c043b806cc3f4b29130d83792a528f75ff1f7c638ba8d19c6b1b56ef02dbced2 +size 223506 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T120851087341Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T120851087341Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a61f8122c4d350ec9d01c92d54b7f0909bb99d22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T120851087341Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 162098741248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 6726708224, + "inactive_split_bytes_peak": 16082528256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 1, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 173111508992, + "reserved_bytes_peak": 182703882240, + "segment_count_current": 827, + "segment_count_peak": 877, + "timestamp_utc": "2026-08-01T12:08:51.086847Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104496814080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162098741248, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104496814080, + "device": 3, + "inactive_split_bytes_current": 10216600576, + "inactive_split_bytes_peak": 16082528256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 181481242624, + "reserved_bytes_peak": 181481242624, + "segment_count_current": 867, + "segment_count_peak": 867, + "timestamp_utc": "2026-08-01T12:08:04.132243Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6726708224, + "inactive_split_bytes_current_delta": -3489892352, + "inactive_split_bytes_peak": 16082528256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 1, + "num_alloc_retries_delta": 1, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 173111508992, + "reserved_bytes_current_delta": -8369733632, + "reserved_bytes_peak": 182703882240, + "reserved_bytes_peak_delta": 1222639616, + "segment_count_current": 827, + "segment_count_current_delta": -40, + "segment_count_peak": 877, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T120851087341Z.pickle", + "timestamp_utc": "2026-08-01T12:08:51.172926Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T120851087341Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T120851087341Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3624dd093cb483160fc7d59c1ee447b15fb59129 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T120851087341Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b69bc6ea6e63dc2e0c589d34563e6acff26fad249ff0db4f65cbb7c4c2d80e6 +size 4039431 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T120754712663Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T120754712663Z.json new file mode 100644 index 0000000000000000000000000000000000000000..598e1aed0282731dc5d4441f2076c5fb3d68db18 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T120754712663Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72458401792, + "active_bytes_current_delta": 37226504192, + "active_bytes_peak": 81192394752, + "active_bytes_peak_delta": 45725616128, + "after": { + "active_bytes_current": 72458401792, + "active_bytes_peak": 81192394752, + "allocated_bytes_current": 72458401792, + "allocated_bytes_peak": 80846758912, + "device": 4, + "inactive_split_bytes_current": 5098473472, + "inactive_split_bytes_peak": 10387943936, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 97970552832, + "reserved_bytes_peak": 97970552832, + "segment_count_current": 465, + "segment_count_peak": 465, + "timestamp_utc": "2026-08-01T12:07:54.712073Z", + "world_size": 8 + }, + "allocated_bytes_current": 72458401792, + "allocated_bytes_current_delta": 37226504192, + "allocated_bytes_peak": 80846758912, + "allocated_bytes_peak_delta": 45379980288, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:06:45.209705Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5098473472, + "inactive_split_bytes_current_delta": 4966096896, + "inactive_split_bytes_peak": 10387943936, + "inactive_split_bytes_peak_delta": 10234534400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 97970552832, + "reserved_bytes_current_delta": 58726547456, + "reserved_bytes_peak": 97970552832, + "reserved_bytes_peak_delta": 58726547456, + "segment_count_current": 465, + "segment_count_current_delta": 394, + "segment_count_peak": 465, + "segment_count_peak_delta": 394, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T120754712663Z.pickle", + "timestamp_utc": "2026-08-01T12:07:54.829328Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T120754712663Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T120754712663Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..87d660fe7784dcf7dab4d45d98c9d467fb811bb9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T120754712663Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09ee9f9c6047f96dfedbfa542ec45aefb8cfbcc6ebe7be7d45e94b7dd42abc51 +size 5951054 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T120755914154Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T120755914154Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7964b21385cce9e9755503d0d6b0dd17215e1d3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T120755914154Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72458401792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104834117120, + "active_bytes_peak_delta": 23641722368, + "after": { + "active_bytes_current": 72458401792, + "active_bytes_peak": 104834117120, + "allocated_bytes_current": 72458401792, + "allocated_bytes_peak": 104485137408, + "device": 4, + "inactive_split_bytes_current": 5947820032, + "inactive_split_bytes_peak": 12379216896, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 110018691072, + "reserved_bytes_peak": 110018691072, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:07:55.913751Z", + "world_size": 8 + }, + "allocated_bytes_current": 72458401792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104485137408, + "allocated_bytes_peak_delta": 23638378496, + "before": { + "active_bytes_current": 72458401792, + "active_bytes_peak": 81192394752, + "allocated_bytes_current": 72458401792, + "allocated_bytes_peak": 80846758912, + "device": 4, + "inactive_split_bytes_current": 5098473472, + "inactive_split_bytes_peak": 10387943936, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 97970552832, + "reserved_bytes_peak": 97970552832, + "segment_count_current": 465, + "segment_count_peak": 465, + "timestamp_utc": "2026-08-01T12:07:54.846341Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5947820032, + "inactive_split_bytes_current_delta": 849346560, + "inactive_split_bytes_peak": 12379216896, + "inactive_split_bytes_peak_delta": 1991272960, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 110018691072, + "reserved_bytes_current_delta": 12048138240, + "reserved_bytes_peak": 110018691072, + "reserved_bytes_peak_delta": 12048138240, + "segment_count_current": 523, + "segment_count_current_delta": 58, + "segment_count_peak": 523, + "segment_count_peak_delta": 58, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T120755914154Z.pickle", + "timestamp_utc": "2026-08-01T12:07:56.035109Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T120755914154Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T120755914154Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cff805b39977507659189fecb5cda6f3af8f594f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T120755914154Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cbaf3dc17f6d098e51eff941628a74c060a1140e41f6fd803c9a4ae22554416 +size 6616855 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T120757012857Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T120757012857Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ffe413e3edd625def9c018e4f77349e0cc1a7679 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T120757012857Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72458401792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104834117120, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72458401792, + "active_bytes_peak": 104834117120, + "allocated_bytes_current": 72458401792, + "allocated_bytes_peak": 104485137408, + "device": 4, + "inactive_split_bytes_current": 5777950720, + "inactive_split_bytes_peak": 12379216896, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 110018691072, + "reserved_bytes_peak": 110018691072, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:07:57.012456Z", + "world_size": 8 + }, + "allocated_bytes_current": 72458401792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104485137408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72458401792, + "active_bytes_peak": 104834117120, + "allocated_bytes_current": 72458401792, + "allocated_bytes_peak": 104485137408, + "device": 4, + "inactive_split_bytes_current": 5947820032, + "inactive_split_bytes_peak": 12379216896, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 110018691072, + "reserved_bytes_peak": 110018691072, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:07:56.049631Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5777950720, + "inactive_split_bytes_current_delta": -169869312, + "inactive_split_bytes_peak": 12379216896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 110018691072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110018691072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T120757012857Z.pickle", + "timestamp_utc": "2026-08-01T12:07:57.124068Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T120757012857Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T120757012857Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7d6aac47260e4580091ca88eadcb9b2a61bbac22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T120757012857Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6aa62f5789989a26e0c32a1196a4ae33c68ff5d2e14957610c5ca594299d96d +size 6596356 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T120759899546Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T120759899546Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61d46b848baefbfa635727bfc15ea89f8edaccbb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T120759899546Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475445248, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162088811520, + "active_bytes_peak_delta": 57254694400, + "after": { + "active_bytes_current": 72475445248, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 72475445248, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_peak": 16934595584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 180810153984, + "reserved_bytes_peak": 180810153984, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-01T12:07:59.899011Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475445248, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104498156544, + "allocated_bytes_peak_delta": 13019136, + "before": { + "active_bytes_current": 72458401792, + "active_bytes_peak": 104834117120, + "allocated_bytes_current": 72458401792, + "allocated_bytes_peak": 104485137408, + "device": 4, + "inactive_split_bytes_current": 5777950720, + "inactive_split_bytes_peak": 12379216896, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 110018691072, + "reserved_bytes_peak": 110018691072, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T12:07:57.137727Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_current_delta": 4697354240, + "inactive_split_bytes_peak": 16934595584, + "inactive_split_bytes_peak_delta": 4555378688, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 180810153984, + "reserved_bytes_current_delta": 70791462912, + "reserved_bytes_peak": 180810153984, + "reserved_bytes_peak_delta": 70791462912, + "segment_count_current": 861, + "segment_count_current_delta": 338, + "segment_count_peak": 861, + "segment_count_peak_delta": 338, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T120759899546Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.026105Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T120759899546Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T120759899546Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7867f9a4a4c258200b1cfb09aae50a94ce16af98 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T120759899546Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22a7e0e8fe1960a8a0c68f1d033af2413a4e42c428b2e01180957170c11c6789 +size 6937566 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T120800855410Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T120800855410Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d6430dd38f8e3e7b8335988f97aa3bd5647adecf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T120800855410Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475445248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162088811520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475445248, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 72475445248, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_peak": 16934595584, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 180810153984, + "reserved_bytes_peak": 180810153984, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-01T12:08:00.854939Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475445248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104498156544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475445248, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 72475445248, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_peak": 16934595584, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 180810153984, + "reserved_bytes_peak": 180810153984, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-01T12:08:00.041704Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16934595584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 180810153984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180810153984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 861, + "segment_count_current_delta": 0, + "segment_count_peak": 861, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T120800855410Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.861585Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T120800855410Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T120800855410Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3db251feb898bfb3815be5329998d450cf4ebb4a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T120800855410Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf2c76484d53b6d31d16321157e257cef7cf17fcfa654b8991dc600a55b106b5 +size 223522 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T120801671881Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T120801671881Z.json new file mode 100644 index 0000000000000000000000000000000000000000..604567ae413d8c27dec5f21b428a5a08ef29e5a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T120801671881Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475445248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162088811520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475445248, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 72475445248, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_peak": 16934595584, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 180810153984, + "reserved_bytes_peak": 180810153984, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-01T12:08:01.671464Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475445248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104498156544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475445248, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 72475445248, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_peak": 16934595584, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 180810153984, + "reserved_bytes_peak": 180810153984, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-01T12:08:00.862533Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16934595584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 180810153984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180810153984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 861, + "segment_count_current_delta": 0, + "segment_count_peak": 861, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T120801671881Z.pickle", + "timestamp_utc": "2026-08-01T12:08:01.677604Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T120801671881Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T120801671881Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..831fa395b501c568ffcfccedbd92347fcd8347c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T120801671881Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f137882b6737db2ce53421151840b66514360ce5fa81c31ff2b621c29a7d9200 +size 223522 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T120802490132Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T120802490132Z.json new file mode 100644 index 0000000000000000000000000000000000000000..39cf4bf4a6af7b15f05c3e2f3bf085132845013a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T120802490132Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475445248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162088811520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475445248, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 72475445248, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_peak": 16934595584, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 180810153984, + "reserved_bytes_peak": 180810153984, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-01T12:08:02.489740Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475445248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104498156544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475445248, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 72475445248, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_peak": 16934595584, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 180810153984, + "reserved_bytes_peak": 180810153984, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-01T12:08:01.678503Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16934595584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 180810153984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180810153984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 861, + "segment_count_current_delta": 0, + "segment_count_peak": 861, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T120802490132Z.pickle", + "timestamp_utc": "2026-08-01T12:08:02.495664Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T120802490132Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T120802490132Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..358657b361d3d534e49276ee11ae63f8383dc9d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T120802490132Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34b6f9220c07ce8016ae55350a08bbbb122aab4808ddddf7d348e489fcf52ec2 +size 223522 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T120803305918Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T120803305918Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3757dac3d4e4bd23c8b067ac4a88b8eea3befc5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T120803305918Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475445248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162088811520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475445248, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 72475445248, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_peak": 16934595584, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 180810153984, + "reserved_bytes_peak": 180810153984, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-01T12:08:03.305533Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475445248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104498156544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475445248, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 72475445248, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_peak": 16934595584, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 180810153984, + "reserved_bytes_peak": 180810153984, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-01T12:08:02.496584Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16934595584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 180810153984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180810153984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 861, + "segment_count_current_delta": 0, + "segment_count_peak": 861, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T120803305918Z.pickle", + "timestamp_utc": "2026-08-01T12:08:03.311322Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T120803305918Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T120803305918Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d69fb69e2114e9a80d77dfa4595541a2d57b5ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T120803305918Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5969185b10694478c942c7dfb4c03d86470725297c80e2578f7e2a300484bf0a +size 223522 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T120804125002Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T120804125002Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5bd807d9f7626d25efa4f6122c2ac103b1a7ee87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T120804125002Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475445248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162088811520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475445248, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 72475445248, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_peak": 16934595584, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 180810153984, + "reserved_bytes_peak": 180810153984, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-01T12:08:04.124593Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475445248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104498156544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475445248, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 72475445248, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_peak": 16934595584, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 180810153984, + "reserved_bytes_peak": 180810153984, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-01T12:08:03.312248Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16934595584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 180810153984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180810153984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 861, + "segment_count_current_delta": 0, + "segment_count_peak": 861, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T120804125002Z.pickle", + "timestamp_utc": "2026-08-01T12:08:04.130803Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T120804125002Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T120804125002Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..70f934a45cb6f635b824865c2786ee8dfd18ca23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T120804125002Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4aeedf6d0679282ac9424368ca9184ada8efad8b2952c0d02a99a182a5a7134 +size 223522 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T120851087346Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T120851087346Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d36599c67cc582f0c53b04c43ee52b5f02b1019 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T120851087346Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61218164736, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 162088811520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61218164736, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 61218164736, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 7073492992, + "inactive_split_bytes_peak": 16934595584, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 1, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 172757090304, + "reserved_bytes_peak": 182737436672, + "segment_count_current": 823, + "segment_count_peak": 874, + "timestamp_utc": "2026-08-01T12:08:51.086862Z", + "world_size": 8 + }, + "allocated_bytes_current": 61218164736, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104498156544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475445248, + "active_bytes_peak": 162088811520, + "allocated_bytes_current": 72475445248, + "allocated_bytes_peak": 104498156544, + "device": 4, + "inactive_split_bytes_current": 10475304960, + "inactive_split_bytes_peak": 16934595584, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 180810153984, + "reserved_bytes_peak": 180810153984, + "segment_count_current": 861, + "segment_count_peak": 861, + "timestamp_utc": "2026-08-01T12:08:04.131728Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7073492992, + "inactive_split_bytes_current_delta": -3401811968, + "inactive_split_bytes_peak": 16934595584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 1, + "num_alloc_retries_delta": 1, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 172757090304, + "reserved_bytes_current_delta": -8053063680, + "reserved_bytes_peak": 182737436672, + "reserved_bytes_peak_delta": 1927282688, + "segment_count_current": 823, + "segment_count_current_delta": -38, + "segment_count_peak": 874, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T120851087346Z.pickle", + "timestamp_utc": "2026-08-01T12:08:51.170041Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T120851087346Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T120851087346Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e0b83897438bfc5eb9be3385d7b31c8ffc6bacf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T120851087346Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63f6fd352dfff37e052adcb4758d92a3af7c140c9b0a58810956586f21af2780 +size 4167495 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T120754712719Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T120754712719Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f561b9a5f5cafe0bd62e8d25383a2645d3c8626 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T120754712719Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 81188724736, + "active_bytes_peak_delta": 45722388480, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 81188724736, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80844988928, + "device": 5, + "inactive_split_bytes_current": 5406000128, + "inactive_split_bytes_peak": 8658412032, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 97899249664, + "reserved_bytes_peak": 97899249664, + "segment_count_current": 469, + "segment_count_peak": 469, + "timestamp_utc": "2026-08-01T12:07:54.712124Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 80844988928, + "allocated_bytes_peak_delta": 45378652672, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:06:45.210176Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5406000128, + "inactive_split_bytes_current_delta": 5273181184, + "inactive_split_bytes_peak": 8658412032, + "inactive_split_bytes_peak_delta": 8498025984, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 97899249664, + "reserved_bytes_current_delta": 58655244288, + "reserved_bytes_peak": 97899249664, + "reserved_bytes_peak_delta": 58655244288, + "segment_count_current": 469, + "segment_count_current_delta": 398, + "segment_count_peak": 469, + "segment_count_peak_delta": 398, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T120754712719Z.pickle", + "timestamp_utc": "2026-08-01T12:07:54.832066Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T120754712719Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T120754712719Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b1d732e3f71b34575d915741b60bf5c64b502dbd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T120754712719Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e974a6cea55b071493b262d049dacf6d925a9a809aaed40df60859f1ae7f862f +size 5942981 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T120755914226Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T120755914226Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f868ddfae11af1590910c874bd3e27d9fdb6e13f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T120755914226Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104831647232, + "active_bytes_peak_delta": 23642922496, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104831647232, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104482820096, + "device": 5, + "inactive_split_bytes_current": 6611862528, + "inactive_split_bytes_peak": 11088428032, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109135790080, + "reserved_bytes_peak": 109135790080, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T12:07:55.913822Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104482820096, + "allocated_bytes_peak_delta": 23637831168, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 81188724736, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80844988928, + "device": 5, + "inactive_split_bytes_current": 5406000128, + "inactive_split_bytes_peak": 8658412032, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 97899249664, + "reserved_bytes_peak": 97899249664, + "segment_count_current": 469, + "segment_count_peak": 469, + "timestamp_utc": "2026-08-01T12:07:54.850473Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6611862528, + "inactive_split_bytes_current_delta": 1205862400, + "inactive_split_bytes_peak": 11088428032, + "inactive_split_bytes_peak_delta": 2430016000, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109135790080, + "reserved_bytes_current_delta": 11236540416, + "reserved_bytes_peak": 109135790080, + "reserved_bytes_peak_delta": 11236540416, + "segment_count_current": 519, + "segment_count_current_delta": 50, + "segment_count_peak": 519, + "segment_count_peak_delta": 50, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T120755914226Z.pickle", + "timestamp_utc": "2026-08-01T12:07:56.025773Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T120755914226Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T120755914226Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fae754808ea5fe45ad741f5d7f8a9aa3c554e223 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T120755914226Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e04b52b5dead318f8b1a5edfa8fe118e50e9ed995c779acc8307a925f563988 +size 6587228 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T120757012899Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T120757012899Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04a08fb325555e3d2d333586d5f3e11d255c236f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T120757012899Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104831647232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104831647232, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104484285440, + "device": 5, + "inactive_split_bytes_current": 6259540992, + "inactive_split_bytes_peak": 11269808128, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109548929024, + "reserved_bytes_peak": 109548929024, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T12:07:57.012499Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104484285440, + "allocated_bytes_peak_delta": 1465344, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104831647232, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104482820096, + "device": 5, + "inactive_split_bytes_current": 6611862528, + "inactive_split_bytes_peak": 11088428032, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 109135790080, + "reserved_bytes_peak": 109135790080, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T12:07:56.039881Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6259540992, + "inactive_split_bytes_current_delta": -352321536, + "inactive_split_bytes_peak": 11269808128, + "inactive_split_bytes_peak_delta": 181380096, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109548929024, + "reserved_bytes_current_delta": 413138944, + "reserved_bytes_peak": 109548929024, + "reserved_bytes_peak_delta": 413138944, + "segment_count_current": 520, + "segment_count_current_delta": 1, + "segment_count_peak": 520, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T120757012899Z.pickle", + "timestamp_utc": "2026-08-01T12:07:57.125441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T120757012899Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T120757012899Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fae165c76216cec572fb8357a28692a94c6596b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T120757012899Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9415462aaf44fdc9e85a92130c4b4340b9f57984eaeaf101bbfb394d6c17812e +size 6593341 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T120759899643Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T120759899643Z.json new file mode 100644 index 0000000000000000000000000000000000000000..43cd482b9a85c056e2236f656ac2d92383b2dc4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T120759899643Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162103525376, + "active_bytes_peak_delta": 57271878144, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_peak": 15980868608, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 181290401792, + "reserved_bytes_peak": 181290401792, + "segment_count_current": 870, + "segment_count_peak": 870, + "timestamp_utc": "2026-08-01T12:07:59.899096Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104503760896, + "allocated_bytes_peak_delta": 19475456, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 104831647232, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104484285440, + "device": 5, + "inactive_split_bytes_current": 6259540992, + "inactive_split_bytes_peak": 11269808128, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 109548929024, + "reserved_bytes_peak": 109548929024, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T12:07:57.139541Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_current_delta": 3732664320, + "inactive_split_bytes_peak": 15980868608, + "inactive_split_bytes_peak_delta": 4711060480, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 181290401792, + "reserved_bytes_current_delta": 71741472768, + "reserved_bytes_peak": 181290401792, + "reserved_bytes_peak_delta": 71741472768, + "segment_count_current": 870, + "segment_count_current_delta": 350, + "segment_count_peak": 870, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T120759899643Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.026185Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T120759899643Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T120759899643Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..22cd8bf30de98970bb2808753b4b4578011fc969 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T120759899643Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c71af26c26f688e90f45a261bee4b80e483ee15701370a6bceef4b3c36a0c0c4 +size 6935489 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T120800855462Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T120800855462Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04f2c651fb956d6621ce26a5b01994a01a42353e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T120800855462Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162103525376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_peak": 15980868608, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 181290401792, + "reserved_bytes_peak": 181290401792, + "segment_count_current": 870, + "segment_count_peak": 870, + "timestamp_utc": "2026-08-01T12:08:00.854992Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104503760896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_peak": 15980868608, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 181290401792, + "reserved_bytes_peak": 181290401792, + "segment_count_current": 870, + "segment_count_peak": 870, + "timestamp_utc": "2026-08-01T12:08:00.041737Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15980868608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 181290401792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181290401792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 870, + "segment_count_current_delta": 0, + "segment_count_peak": 870, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T120800855462Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.861479Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T120800855462Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T120800855462Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f54aaf662d28f42f967ff27fbde1e24ac304b979 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T120800855462Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cc80eab994b2b4f031b2b08f32803857b60ecd581dbb24a8512ef4ad3ee93f6 +size 223284 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T120801671885Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T120801671885Z.json new file mode 100644 index 0000000000000000000000000000000000000000..19faa19f31ddf5f049075c118159813a7e4e0426 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T120801671885Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162103525376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_peak": 15980868608, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 181290401792, + "reserved_bytes_peak": 181290401792, + "segment_count_current": 870, + "segment_count_peak": 870, + "timestamp_utc": "2026-08-01T12:08:01.671493Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104503760896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_peak": 15980868608, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 181290401792, + "reserved_bytes_peak": 181290401792, + "segment_count_current": 870, + "segment_count_peak": 870, + "timestamp_utc": "2026-08-01T12:08:00.862420Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15980868608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 181290401792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181290401792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 870, + "segment_count_current_delta": 0, + "segment_count_peak": 870, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T120801671885Z.pickle", + "timestamp_utc": "2026-08-01T12:08:01.677460Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T120801671885Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T120801671885Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2934c3d9c8efda192a16f30b4fe4d124bfecd31e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T120801671885Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf12095c4ae9a362b3115d4aac3d5f5defafd34a3875a2e5d18b959c9ee2346f +size 223253 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T120802490155Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T120802490155Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b608dd73cb731fde3457c75d4ba589de81181f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T120802490155Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162103525376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_peak": 15980868608, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 181290401792, + "reserved_bytes_peak": 181290401792, + "segment_count_current": 870, + "segment_count_peak": 870, + "timestamp_utc": "2026-08-01T12:08:02.489773Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104503760896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_peak": 15980868608, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 181290401792, + "reserved_bytes_peak": 181290401792, + "segment_count_current": 870, + "segment_count_peak": 870, + "timestamp_utc": "2026-08-01T12:08:01.678377Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15980868608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 181290401792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181290401792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 870, + "segment_count_current_delta": 0, + "segment_count_peak": 870, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T120802490155Z.pickle", + "timestamp_utc": "2026-08-01T12:08:02.495545Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T120802490155Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T120802490155Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cfa8cd2ee6d17f5aa496963dbebd448a0ab8e096 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T120802490155Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4fcc1a33600545c48bbd782bce510689bac2e37c3690544a6116450305e2837 +size 223284 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T120803305941Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T120803305941Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f27579a6f3a76a28a8a5146d0f9d31607b6cfdca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T120803305941Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162103525376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_peak": 15980868608, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 181290401792, + "reserved_bytes_peak": 181290401792, + "segment_count_current": 870, + "segment_count_peak": 870, + "timestamp_utc": "2026-08-01T12:08:03.305561Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104503760896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_peak": 15980868608, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 181290401792, + "reserved_bytes_peak": 181290401792, + "segment_count_current": 870, + "segment_count_peak": 870, + "timestamp_utc": "2026-08-01T12:08:02.496479Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15980868608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 181290401792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181290401792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 870, + "segment_count_current_delta": 0, + "segment_count_peak": 870, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T120803305941Z.pickle", + "timestamp_utc": "2026-08-01T12:08:03.311197Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T120803305941Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T120803305941Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6cf1f53bf443958e6320e70e3a08ca81223de273 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T120803305941Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1100e64516478e7d27a1ccf3f4d4bdd6336697b39dc7e585cb5bfffefb8aa10 +size 223253 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T120804124941Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T120804124941Z.json new file mode 100644 index 0000000000000000000000000000000000000000..695c134f32d1448337c8090f5138bc6e054ef832 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T120804124941Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162103525376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_peak": 15980868608, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 181290401792, + "reserved_bytes_peak": 181290401792, + "segment_count_current": 870, + "segment_count_peak": 870, + "timestamp_utc": "2026-08-01T12:08:04.124553Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104503760896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_peak": 15980868608, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 181290401792, + "reserved_bytes_peak": 181290401792, + "segment_count_current": 870, + "segment_count_peak": 870, + "timestamp_utc": "2026-08-01T12:08:03.312090Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15980868608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 181290401792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181290401792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 870, + "segment_count_current_delta": 0, + "segment_count_peak": 870, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T120804124941Z.pickle", + "timestamp_utc": "2026-08-01T12:08:04.130282Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T120804124941Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T120804124941Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3c6a7e16176bd8072baabe251da468b088b7a03c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T120804124941Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b12aaebee1502531adf7d70fad5cd3448a90c58584e59bb3ded840a62dd806b +size 223284 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T120851087335Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T120851087335Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e474644573e3296c8f97a076e2e117de48526d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T120851087335Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 162103525376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 6561033216, + "inactive_split_bytes_peak": 15980868608, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 1, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 172947931136, + "reserved_bytes_peak": 182747922432, + "segment_count_current": 829, + "segment_count_peak": 881, + "timestamp_utc": "2026-08-01T12:08:51.086855Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104503760896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162103525376, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104503760896, + "device": 5, + "inactive_split_bytes_current": 9992205312, + "inactive_split_bytes_peak": 15980868608, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 181290401792, + "reserved_bytes_peak": 181290401792, + "segment_count_current": 870, + "segment_count_peak": 870, + "timestamp_utc": "2026-08-01T12:08:04.131196Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6561033216, + "inactive_split_bytes_current_delta": -3431172096, + "inactive_split_bytes_peak": 15980868608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 1, + "num_alloc_retries_delta": 1, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 172947931136, + "reserved_bytes_current_delta": -8342470656, + "reserved_bytes_peak": 182747922432, + "reserved_bytes_peak_delta": 1457520640, + "segment_count_current": 829, + "segment_count_current_delta": -41, + "segment_count_peak": 881, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T120851087335Z.pickle", + "timestamp_utc": "2026-08-01T12:08:51.172835Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T120851087335Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T120851087335Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13554dd8e83c983bb4ef424e182dfe418081a35d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T120851087335Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6a9a24c401160700fb1fcf6bebd1356385accdf34a98d638a5029aedf837cd0 +size 4040102 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T120754712705Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T120754712705Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9810c2697a50149737b35a670a8a1dad6166e887 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T120754712705Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 81185269248, + "active_bytes_peak_delta": 45718932992, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 81185269248, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80845578752, + "device": 6, + "inactive_split_bytes_current": 5435360256, + "inactive_split_bytes_peak": 9318023680, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 97750351872, + "reserved_bytes_peak": 97750351872, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T12:07:54.712105Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 80845578752, + "allocated_bytes_peak_delta": 45379242496, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:06:45.210090Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5435360256, + "inactive_split_bytes_current_delta": 5302541312, + "inactive_split_bytes_peak": 9318023680, + "inactive_split_bytes_peak_delta": 9157637632, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 97750351872, + "reserved_bytes_current_delta": 58506346496, + "reserved_bytes_peak": 97750351872, + "reserved_bytes_peak_delta": 58506346496, + "segment_count_current": 466, + "segment_count_current_delta": 395, + "segment_count_peak": 466, + "segment_count_peak_delta": 395, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T120754712705Z.pickle", + "timestamp_utc": "2026-08-01T12:07:54.831684Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T120754712705Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T120754712705Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9df3faf85bda584bf6e2fca78c44802505273860 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T120754712705Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dde5839fafa1bf3b8b6170a3f3cb80315adadea29d59f9c0635da2fe7427fc5 +size 5942559 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T120755914222Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T120755914222Z.json new file mode 100644 index 0000000000000000000000000000000000000000..463e96554938e4b4aa0e38cc32ad3173ca808bfd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T120755914222Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105172144640, + "active_bytes_peak_delta": 23986875392, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105172144640, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104482799616, + "device": 6, + "inactive_split_bytes_current": 6322455552, + "inactive_split_bytes_peak": 11321629696, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109400031232, + "reserved_bytes_peak": 109400031232, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T12:07:55.913811Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104482799616, + "allocated_bytes_peak_delta": 23637220864, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 81185269248, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 80845578752, + "device": 6, + "inactive_split_bytes_current": 5435360256, + "inactive_split_bytes_peak": 9318023680, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 97750351872, + "reserved_bytes_peak": 97750351872, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T12:07:54.848510Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6322455552, + "inactive_split_bytes_current_delta": 887095296, + "inactive_split_bytes_peak": 11321629696, + "inactive_split_bytes_peak_delta": 2003606016, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109400031232, + "reserved_bytes_current_delta": 11649679360, + "reserved_bytes_peak": 109400031232, + "reserved_bytes_peak_delta": 11649679360, + "segment_count_current": 519, + "segment_count_current_delta": 53, + "segment_count_peak": 519, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T120755914222Z.pickle", + "timestamp_utc": "2026-08-01T12:07:56.028824Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T120755914222Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T120755914222Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e280685bf95e45b010d97ded8cbb84639e9bab24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T120755914222Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e676a4198591b2adb0bca9870748aeb05e8f72bd4aa7d35e7a53357daf88f2bb +size 6058695 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T120757012892Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T120757012892Z.json new file mode 100644 index 0000000000000000000000000000000000000000..74272219015166d45fd2e96ab07f58905b6e22b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T120757012892Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105172144640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105172144640, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104484505600, + "device": 6, + "inactive_split_bytes_current": 5995299840, + "inactive_split_bytes_peak": 12166467584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109813170176, + "reserved_bytes_peak": 109813170176, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T12:07:57.012498Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104484505600, + "allocated_bytes_peak_delta": 1705984, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105172144640, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104482799616, + "device": 6, + "inactive_split_bytes_current": 6322455552, + "inactive_split_bytes_peak": 11321629696, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 109400031232, + "reserved_bytes_peak": 109400031232, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T12:07:56.042732Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5995299840, + "inactive_split_bytes_current_delta": -327155712, + "inactive_split_bytes_peak": 12166467584, + "inactive_split_bytes_peak_delta": 844837888, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109813170176, + "reserved_bytes_current_delta": 413138944, + "reserved_bytes_peak": 109813170176, + "reserved_bytes_peak_delta": 413138944, + "segment_count_current": 520, + "segment_count_current_delta": 1, + "segment_count_peak": 520, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T120757012892Z.pickle", + "timestamp_utc": "2026-08-01T12:07:57.127031Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T120757012892Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T120757012892Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba049418fb5202eca84fbccd069e3383e122f7f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T120757012892Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6917524c180f572ce944985059abdd331b86c59392138702961525503a801aa1 +size 6578661 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T120759899633Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T120759899633Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d705444f55a78a79c3523b2f428fd075bb2ac0f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T120759899633Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162099920896, + "active_bytes_peak_delta": 56927776256, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_peak": 16145442816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 180883554304, + "reserved_bytes_peak": 180883554304, + "segment_count_current": 864, + "segment_count_peak": 864, + "timestamp_utc": "2026-08-01T12:07:59.899073Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104502384640, + "allocated_bytes_peak_delta": 17879040, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105172144640, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 104484505600, + "device": 6, + "inactive_split_bytes_current": 5995299840, + "inactive_split_bytes_peak": 12166467584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 109813170176, + "reserved_bytes_peak": 109813170176, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T12:07:57.140767Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_current_delta": 4498124800, + "inactive_split_bytes_peak": 16145442816, + "inactive_split_bytes_peak_delta": 3978975232, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 180883554304, + "reserved_bytes_current_delta": 71070384128, + "reserved_bytes_peak": 180883554304, + "reserved_bytes_peak_delta": 71070384128, + "segment_count_current": 864, + "segment_count_current_delta": 344, + "segment_count_peak": 864, + "segment_count_peak_delta": 344, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T120759899633Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.025225Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T120759899633Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T120759899633Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8722375388c3502b603091a0175d18b855685be0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T120759899633Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d4e4282b47198a40b2f2a5a743621a1e6b5d693be491841e25a6ebe5f82f9ce +size 6935910 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T120800855439Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T120800855439Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4aff17b6e8191aa5a270005c71146863952bc47e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T120800855439Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162099920896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_peak": 16145442816, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 180883554304, + "reserved_bytes_peak": 180883554304, + "segment_count_current": 864, + "segment_count_peak": 864, + "timestamp_utc": "2026-08-01T12:08:00.854960Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104502384640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_peak": 16145442816, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 180883554304, + "reserved_bytes_peak": 180883554304, + "segment_count_current": 864, + "segment_count_peak": 864, + "timestamp_utc": "2026-08-01T12:08:00.040339Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16145442816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 180883554304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180883554304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 864, + "segment_count_current_delta": 0, + "segment_count_peak": 864, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T120800855439Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.861428Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T120800855439Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T120800855439Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..072296fd23f93f7cad83c9514bcf7b3bfb9694e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T120800855439Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a28f3183b2c026cccaf70241d3147041d9e81f55fe007a1033c65b284017cc05 +size 221800 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T120801671894Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T120801671894Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a7de2048fcf1874b23c088b8ed0a800f51a4017 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T120801671894Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162099920896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_peak": 16145442816, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 180883554304, + "reserved_bytes_peak": 180883554304, + "segment_count_current": 864, + "segment_count_peak": 864, + "timestamp_utc": "2026-08-01T12:08:01.671492Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104502384640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_peak": 16145442816, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 180883554304, + "reserved_bytes_peak": 180883554304, + "segment_count_current": 864, + "segment_count_peak": 864, + "timestamp_utc": "2026-08-01T12:08:00.862378Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16145442816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 180883554304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180883554304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 864, + "segment_count_current_delta": 0, + "segment_count_peak": 864, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T120801671894Z.pickle", + "timestamp_utc": "2026-08-01T12:08:01.677517Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T120801671894Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T120801671894Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93917992a1f6ca449a7997cd352cc906accbd1a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T120801671894Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c73675551eaa1a45e76c4f487187674025d9d2ff5785533e8c387d623e67da4 +size 221769 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T120802490189Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T120802490189Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b5be0a830b3372bb8eef3476e541e175fc9baaa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T120802490189Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162099920896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_peak": 16145442816, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 180883554304, + "reserved_bytes_peak": 180883554304, + "segment_count_current": 864, + "segment_count_peak": 864, + "timestamp_utc": "2026-08-01T12:08:02.489789Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104502384640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_peak": 16145442816, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 180883554304, + "reserved_bytes_peak": 180883554304, + "segment_count_current": 864, + "segment_count_peak": 864, + "timestamp_utc": "2026-08-01T12:08:01.678420Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16145442816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 180883554304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180883554304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 864, + "segment_count_current_delta": 0, + "segment_count_peak": 864, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T120802490189Z.pickle", + "timestamp_utc": "2026-08-01T12:08:02.495739Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T120802490189Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T120802490189Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f902fcd034b127120c694b3a7fead01b3fc5d978 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T120802490189Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:719e2fddfd576c382e611631bd435d34641991cbfdea944e58f38fd18995422e +size 221800 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T120803305977Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T120803305977Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f294484b48aca728f7fe2b3bc28ec2db3915e69 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T120803305977Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162099920896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_peak": 16145442816, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 180883554304, + "reserved_bytes_peak": 180883554304, + "segment_count_current": 864, + "segment_count_peak": 864, + "timestamp_utc": "2026-08-01T12:08:03.305596Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104502384640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_peak": 16145442816, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 180883554304, + "reserved_bytes_peak": 180883554304, + "segment_count_current": 864, + "segment_count_peak": 864, + "timestamp_utc": "2026-08-01T12:08:02.496642Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16145442816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 180883554304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180883554304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 864, + "segment_count_current_delta": 0, + "segment_count_peak": 864, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T120803305977Z.pickle", + "timestamp_utc": "2026-08-01T12:08:03.311449Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T120803305977Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T120803305977Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..78e6b4c25da9af460485588886cce501ae6c91f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T120803305977Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48a961d2c6b7de07ac64ecff6da1a05d184d9b44e9f8aa783b1ee0ee9ef17784 +size 221769 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T120804124939Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T120804124939Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5837770194fbf1db2435c7ba26b3c95a803aff0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T120804124939Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162099920896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_peak": 16145442816, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 180883554304, + "reserved_bytes_peak": 180883554304, + "segment_count_current": 864, + "segment_count_peak": 864, + "timestamp_utc": "2026-08-01T12:08:04.124541Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104502384640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_peak": 16145442816, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 180883554304, + "reserved_bytes_peak": 180883554304, + "segment_count_current": 864, + "segment_count_peak": 864, + "timestamp_utc": "2026-08-01T12:08:03.312357Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16145442816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 180883554304, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 180883554304, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 864, + "segment_count_current_delta": 0, + "segment_count_peak": 864, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T120804124939Z.pickle", + "timestamp_utc": "2026-08-01T12:08:04.130333Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T120804124939Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T120804124939Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..008c8946af6502479ae772b9c7e84bf538bc749a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T120804124939Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72451d8a7d04f00cc75c475e6458756a1a218ee78df8ab084caeaf060689a44a +size 221800 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T120851087329Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T120851087329Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ca39cf96bfcea173fdc130b06df88c53a5eaf76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T120851087329Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 162099920896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 7003532288, + "inactive_split_bytes_peak": 16145442816, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 1, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 173245726720, + "reserved_bytes_peak": 182810836992, + "segment_count_current": 826, + "segment_count_peak": 877, + "timestamp_utc": "2026-08-01T12:08:51.086855Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104502384640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 162099920896, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 104502384640, + "device": 6, + "inactive_split_bytes_current": 10493424640, + "inactive_split_bytes_peak": 16145442816, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 180883554304, + "reserved_bytes_peak": 180883554304, + "segment_count_current": 864, + "segment_count_peak": 864, + "timestamp_utc": "2026-08-01T12:08:04.131245Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7003532288, + "inactive_split_bytes_current_delta": -3489892352, + "inactive_split_bytes_peak": 16145442816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 1, + "num_alloc_retries_delta": 1, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 173245726720, + "reserved_bytes_current_delta": -7637827584, + "reserved_bytes_peak": 182810836992, + "reserved_bytes_peak_delta": 1927282688, + "segment_count_current": 826, + "segment_count_current_delta": -38, + "segment_count_peak": 877, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T120851087329Z.pickle", + "timestamp_utc": "2026-08-01T12:08:51.175609Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T120851087329Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T120851087329Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c18358264eaa1fe049e48c962959fabd74fdadc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T120851087329Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4320ab35ee8bdc1192342861d7a7523de41f6e0fae1fc923a90f5ae5b8ed3ab +size 4038320 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T120754712704Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T120754712704Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b72cf2233d42cc23fb12a8cec00dbf6967256da2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T120754712704Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059840, + "active_bytes_current_delta": 37225604608, + "active_bytes_peak": 81177864192, + "active_bytes_peak_delta": 45711527936, + "after": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 81177864192, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 80838500352, + "device": 7, + "inactive_split_bytes_current": 5036900864, + "inactive_split_bytes_peak": 10100653568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 98662612992, + "reserved_bytes_peak": 98662612992, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:07:54.712110Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059840, + "allocated_bytes_current_delta": 37225604608, + "allocated_bytes_peak": 80838500352, + "allocated_bytes_peak_delta": 45372164096, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:06:45.210023Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5036900864, + "inactive_split_bytes_current_delta": 4904081920, + "inactive_split_bytes_peak": 10100653568, + "inactive_split_bytes_peak_delta": 9940267520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 98662612992, + "reserved_bytes_current_delta": 59418607616, + "reserved_bytes_peak": 98662612992, + "reserved_bytes_peak_delta": 59418607616, + "segment_count_current": 467, + "segment_count_current_delta": 396, + "segment_count_peak": 467, + "segment_count_peak_delta": 396, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T120754712704Z.pickle", + "timestamp_utc": "2026-08-01T12:07:54.831004Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T120754712704Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T120754712704Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..be3a5f8b84394e570e889448e2a6e9bf840867d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T120754712704Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6487916e2af750f41ece113abb68e592ed08d3b24f842bd9d20a1344fd112590 +size 5944403 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T120755914329Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T120755914329Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2575dbb2e2a891749b18eceb10418d0c94054483 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T120755914329Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104820175360, + "active_bytes_peak_delta": 23642311168, + "after": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 104820175360, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 104484175872, + "device": 7, + "inactive_split_bytes_current": 6563627520, + "inactive_split_bytes_peak": 12175207936, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 109605552128, + "reserved_bytes_peak": 109605552128, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T12:07:55.913920Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104484175872, + "allocated_bytes_peak_delta": 23645675520, + "before": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 81177864192, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 80838500352, + "device": 7, + "inactive_split_bytes_current": 5036900864, + "inactive_split_bytes_peak": 10100653568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 98662612992, + "reserved_bytes_peak": 98662612992, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:07:54.850045Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6563627520, + "inactive_split_bytes_current_delta": 1526726656, + "inactive_split_bytes_peak": 12175207936, + "inactive_split_bytes_peak_delta": 2074554368, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 109605552128, + "reserved_bytes_current_delta": 10942939136, + "reserved_bytes_peak": 109605552128, + "reserved_bytes_peak_delta": 10942939136, + "segment_count_current": 517, + "segment_count_current_delta": 50, + "segment_count_peak": 517, + "segment_count_peak_delta": 50, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T120755914329Z.pickle", + "timestamp_utc": "2026-08-01T12:07:56.029145Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T120755914329Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T120755914329Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..761dbee800e89366c6899a539864dbfe4b297a70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T120755914329Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d46c3164926f03b28d8e2fae01d5a1e59f2d900619e2b42f31bb14b4f9a0eaa +size 6004097 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T120757012894Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T120757012894Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c287019a1afb40aae775c25186304937be88963 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T120757012894Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104826663424, + "active_bytes_peak_delta": 6488064, + "after": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 104826663424, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 104486383104, + "device": 7, + "inactive_split_bytes_current": 6337135104, + "inactive_split_bytes_peak": 12175207936, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 109605552128, + "reserved_bytes_peak": 109605552128, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T12:07:57.012483Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104486383104, + "allocated_bytes_peak_delta": 2207232, + "before": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 104820175360, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 104484175872, + "device": 7, + "inactive_split_bytes_current": 6563627520, + "inactive_split_bytes_peak": 12175207936, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 109605552128, + "reserved_bytes_peak": 109605552128, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T12:07:56.047597Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6337135104, + "inactive_split_bytes_current_delta": -226492416, + "inactive_split_bytes_peak": 12175207936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 109605552128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109605552128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 517, + "segment_count_current_delta": 0, + "segment_count_peak": 517, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T120757012894Z.pickle", + "timestamp_utc": "2026-08-01T12:07:57.123121Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T120757012894Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T120757012894Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ca44add06ffaabb0458b26c1562fa439c36723e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T120757012894Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbeb70a136f7369d9c2a05a5f2b51aa36f7f68a9283ec946f799f7963f42af2f +size 6586341 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T120759899755Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T120759899755Z.json new file mode 100644 index 0000000000000000000000000000000000000000..018d641ceb2ad89ee22c5f55e6550bb9d0360ba7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T120759899755Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162097627648, + "active_bytes_peak_delta": 57270964224, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 11120472576, + "inactive_split_bytes_peak": 16615925248, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 181367996416, + "reserved_bytes_peak": 181367996416, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-01T12:07:59.899171Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 104496814592, + "allocated_bytes_peak_delta": 10431488, + "before": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 104826663424, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 104486383104, + "device": 7, + "inactive_split_bytes_current": 6337135104, + "inactive_split_bytes_peak": 12175207936, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 109605552128, + "reserved_bytes_peak": 109605552128, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T12:07:57.138627Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11120472576, + "inactive_split_bytes_current_delta": 4783337472, + "inactive_split_bytes_peak": 16615925248, + "inactive_split_bytes_peak_delta": 4440717312, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 181367996416, + "reserved_bytes_current_delta": 71762444288, + "reserved_bytes_peak": 181367996416, + "reserved_bytes_peak_delta": 71762444288, + "segment_count_current": 863, + "segment_count_current_delta": 346, + "segment_count_peak": 863, + "segment_count_peak_delta": 346, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T120759899755Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.027921Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T120759899755Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T120759899755Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7b433d85361b11eba8133175303e6236704c95bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T120759899755Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4000f37cbda5dfccfa1efc896020e767d858c804ec4ca7c3bbfe8afd92a87d74 +size 6938201 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T120800855504Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T120800855504Z.json new file mode 100644 index 0000000000000000000000000000000000000000..28cdefa68bd4a833a29948953db4bef8c35fc798 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T120800855504Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162097627648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_peak": 16615925248, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 181367996416, + "reserved_bytes_peak": 181367996416, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-01T12:08:00.855009Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496814592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 11120472576, + "inactive_split_bytes_peak": 16615925248, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 181367996416, + "reserved_bytes_peak": 181367996416, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-01T12:08:00.045490Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 16615925248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 181367996416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181367996416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 863, + "segment_count_current_delta": 0, + "segment_count_peak": 863, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T120800855504Z.pickle", + "timestamp_utc": "2026-08-01T12:08:00.861660Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T120800855504Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T120800855504Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5de0d2a9b9502f9a32ec59d76b975c02158d13e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T120800855504Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bea3f742d94b01a3857feef6969fb47e3fda784240a60e80bb8b334611df762 +size 224383 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T120801672109Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T120801672109Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d5ea3d29e7a0a0ba6cbc5ff2a8ccc61f148f0590 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T120801672109Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162097627648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_peak": 16615925248, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 181367996416, + "reserved_bytes_peak": 181367996416, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-01T12:08:01.671694Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496814592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_peak": 16615925248, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 181367996416, + "reserved_bytes_peak": 181367996416, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-01T12:08:00.862577Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16615925248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 181367996416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181367996416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 863, + "segment_count_current_delta": 0, + "segment_count_peak": 863, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T120801672109Z.pickle", + "timestamp_utc": "2026-08-01T12:08:01.679681Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T120801672109Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T120801672109Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f6a776c6d5529726c5f40abc8962ed9226656b41 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T120801672109Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c884625a9d09b2308e17c187ba17a595aa1661b6d00ab0ed8acab668a20ba68b +size 224352 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T120802490222Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T120802490222Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d0aad797302e669b2bd1558a960a66f5d06ddd3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T120802490222Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162097627648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_peak": 16615925248, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 181367996416, + "reserved_bytes_peak": 181367996416, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-01T12:08:02.489822Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496814592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_peak": 16615925248, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 181367996416, + "reserved_bytes_peak": 181367996416, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-01T12:08:01.680579Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16615925248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 181367996416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181367996416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 863, + "segment_count_current_delta": 0, + "segment_count_peak": 863, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T120802490222Z.pickle", + "timestamp_utc": "2026-08-01T12:08:02.495664Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T120802490222Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T120802490222Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..312898acf5cca443e7dc7c5e59071d9d0dc056ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T120802490222Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a75005ec2b581ae71a2d6a681310e20dc2a2be6233c2da554e9b3c12b3ba1538 +size 224383 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T120803306000Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T120803306000Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5188045d7e53035f96331f801165fe0ce15be278 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T120803306000Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162097627648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_peak": 16615925248, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 181367996416, + "reserved_bytes_peak": 181367996416, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-01T12:08:03.305615Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496814592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_peak": 16615925248, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 181367996416, + "reserved_bytes_peak": 181367996416, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-01T12:08:02.496544Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16615925248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 181367996416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181367996416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 863, + "segment_count_current_delta": 0, + "segment_count_peak": 863, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T120803306000Z.pickle", + "timestamp_utc": "2026-08-01T12:08:03.311320Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T120803306000Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T120803306000Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0599f23a6d6a50506bcf1ef436bb5b377f35f9d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T120803306000Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:101fd8bb1ef9393f9d29f8fdc380dccbf9326969a874b6b62099f0775ad953fa +size 224383 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T120804124960Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T120804124960Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5233461847cc5a04d7ca8f0dfe8e5e5520a62d55 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T120804124960Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162097627648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_peak": 16615925248, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 181367996416, + "reserved_bytes_peak": 181367996416, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-01T12:08:04.124568Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104496814592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_peak": 16615925248, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 181367996416, + "reserved_bytes_peak": 181367996416, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-01T12:08:03.312198Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16615925248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 181367996416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181367996416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 863, + "segment_count_current_delta": 0, + "segment_count_peak": 863, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T120804124960Z.pickle", + "timestamp_utc": "2026-08-01T12:08:04.130510Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T120804124960Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T120804124960Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a87638c78c1bf37fc78d163135c8a5ec4290d86c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T120804124960Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa9b35c7a319e3c4523ce6377acdfc2c1f7c9023572168197ba4c7253149649f +size 224352 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T120851087342Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T120851087342Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e3711c11184e93527eb529f799b477b80b17958c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T120851087342Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822784, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 162097627648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822784, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 61216822784, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 7599122944, + "inactive_split_bytes_peak": 16615925248, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 1, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 173986021376, + "reserved_bytes_peak": 182825517056, + "segment_count_current": 826, + "segment_count_peak": 874, + "timestamp_utc": "2026-08-01T12:08:51.086883Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822784, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 104496814592, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 162097627648, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 104496814592, + "device": 7, + "inactive_split_bytes_current": 11118375424, + "inactive_split_bytes_peak": 16615925248, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 181367996416, + "reserved_bytes_peak": 181367996416, + "segment_count_current": 863, + "segment_count_peak": 863, + "timestamp_utc": "2026-08-01T12:08:04.131412Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7599122944, + "inactive_split_bytes_current_delta": -3519252480, + "inactive_split_bytes_peak": 16615925248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 1, + "num_alloc_retries_delta": 1, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 173986021376, + "reserved_bytes_current_delta": -7381975040, + "reserved_bytes_peak": 182825517056, + "reserved_bytes_peak_delta": 1457520640, + "segment_count_current": 826, + "segment_count_current_delta": -37, + "segment_count_peak": 874, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T120851087342Z.pickle", + "timestamp_utc": "2026-08-01T12:08:51.170677Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T120851087342Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T120851087342Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ad98f77cad22d87678eb6c4e6aa05ee843ae5d02 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T120851087342Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e011b0c0f2370f6fd4d3ad46ba089d7e2720aced66f8a25d581b2c6d5acede4e +size 4040959 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..6cc2490f796a289c5dea88e961ebbe0bca4afe60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 2048 +capacity: 1.50 +expert_dispatch_overlap: off +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6008 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 231 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T121149508115Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T121149508115Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4cdf37971c441fe0ed960430528a0570d4a924e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T121149508115Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871223296, + "active_bytes_current_delta": 31639768064, + "active_bytes_peak": 82193012224, + "active_bytes_peak_delta": 46726675968, + "after": { + "active_bytes_current": 66871223296, + "active_bytes_peak": 82193012224, + "allocated_bytes_current": 66871223296, + "allocated_bytes_peak": 82193012224, + "device": 0, + "inactive_split_bytes_current": 5092547584, + "inactive_split_bytes_peak": 6890230784, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 97490305024, + "reserved_bytes_peak": 97490305024, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T12:11:49.507646Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871223296, + "allocated_bytes_current_delta": 31639768064, + "allocated_bytes_peak": 82193012224, + "allocated_bytes_peak_delta": 46726675968, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:10:41.423197Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5092547584, + "inactive_split_bytes_current_delta": 4957631488, + "inactive_split_bytes_peak": 6890230784, + "inactive_split_bytes_peak_delta": 6729598976, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 97490305024, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97490305024, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 489, + "segment_count_current_delta": 417, + "segment_count_peak": 489, + "segment_count_peak_delta": 417, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T121149508115Z.pickle", + "timestamp_utc": "2026-08-01T12:11:49.619369Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T121149508115Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T121149508115Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ee0fecb144dc3f7035792204280ed1d73330050d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T121149508115Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d60882f0ebf9fada980f5a37b36069a964dccaa406ffcb893d385563d0b54b58 +size 5382643 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T121150684028Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T121150684028Z.json new file mode 100644 index 0000000000000000000000000000000000000000..872f3c17ab2992981a30200217b901d66bd0ea12 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T121150684028Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871223296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105821298688, + "active_bytes_peak_delta": 23628286464, + "after": { + "active_bytes_current": 66871223296, + "active_bytes_peak": 105821298688, + "allocated_bytes_current": 66871223296, + "allocated_bytes_peak": 105821298688, + "device": 0, + "inactive_split_bytes_current": 5096741888, + "inactive_split_bytes_peak": 6890230784, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107055415296, + "reserved_bytes_peak": 107055415296, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:11:50.683613Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871223296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105821298688, + "allocated_bytes_peak_delta": 23628286464, + "before": { + "active_bytes_current": 66871223296, + "active_bytes_peak": 82193012224, + "allocated_bytes_current": 66871223296, + "allocated_bytes_peak": 82193012224, + "device": 0, + "inactive_split_bytes_current": 5092547584, + "inactive_split_bytes_peak": 6890230784, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 97490305024, + "reserved_bytes_peak": 97490305024, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T12:11:49.636969Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5096741888, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 6890230784, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107055415296, + "reserved_bytes_current_delta": 9565110272, + "reserved_bytes_peak": 107055415296, + "reserved_bytes_peak_delta": 9565110272, + "segment_count_current": 547, + "segment_count_current_delta": 58, + "segment_count_peak": 547, + "segment_count_peak_delta": 58, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T121150684028Z.pickle", + "timestamp_utc": "2026-08-01T12:11:50.791643Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T121150684028Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T121150684028Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b4d70564db3cc1f43725952bd96604dbc11be079 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T121150684028Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5908f9f6ed55b3c83832c39f47dcd8559a29486c8a4e9eefec20cf2c722b30 +size 5559198 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T121151890096Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T121151890096Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e7a639057b702cfa6bad52e03cee22c7cb653c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T121151890096Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871223296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105827301376, + "active_bytes_peak_delta": 6002688, + "after": { + "active_bytes_current": 66871223296, + "active_bytes_peak": 105827301376, + "allocated_bytes_current": 66871223296, + "allocated_bytes_peak": 105827301376, + "device": 0, + "inactive_split_bytes_current": 5744761856, + "inactive_split_bytes_peak": 6890230784, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107055415296, + "reserved_bytes_peak": 107055415296, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:11:51.889670Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871223296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105827301376, + "allocated_bytes_peak_delta": 6002688, + "before": { + "active_bytes_current": 66871223296, + "active_bytes_peak": 105821298688, + "allocated_bytes_current": 66871223296, + "allocated_bytes_peak": 105821298688, + "device": 0, + "inactive_split_bytes_current": 5096741888, + "inactive_split_bytes_peak": 6890230784, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107055415296, + "reserved_bytes_peak": 107055415296, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:11:50.805381Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5744761856, + "inactive_split_bytes_current_delta": 648019968, + "inactive_split_bytes_peak": 6890230784, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107055415296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107055415296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T121151890096Z.pickle", + "timestamp_utc": "2026-08-01T12:11:51.994617Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T121151890096Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T121151890096Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6bff00b96cb87da42a465ad810eaa70524bfef40 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T121151890096Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b045f8841616d172f50ab654847cafd48ff6379fbc4fc376ba53e10036e8e51 +size 5560778 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T121154485645Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T121154485645Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c40e5d4f16e080e59be6538974b94cdce7238f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T121154485645Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888266752, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123989592064, + "active_bytes_peak_delta": 18162290688, + "after": { + "active_bytes_current": 66888266752, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66888266752, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_peak": 8376399872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_peak": 136753184768, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:11:54.485096Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888266752, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105835963904, + "allocated_bytes_peak_delta": 8662528, + "before": { + "active_bytes_current": 66871223296, + "active_bytes_peak": 105827301376, + "allocated_bytes_current": 66871223296, + "allocated_bytes_peak": 105827301376, + "device": 0, + "inactive_split_bytes_current": 5744761856, + "inactive_split_bytes_peak": 6890230784, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107055415296, + "reserved_bytes_peak": 107055415296, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:11:52.007678Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_current_delta": -222564352, + "inactive_split_bytes_peak": 8376399872, + "inactive_split_bytes_peak_delta": 1486169088, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_current_delta": 29697769472, + "reserved_bytes_peak": 136753184768, + "reserved_bytes_peak_delta": 29697769472, + "segment_count_current": 752, + "segment_count_current_delta": 205, + "segment_count_peak": 752, + "segment_count_peak_delta": 205, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T121154485645Z.pickle", + "timestamp_utc": "2026-08-01T12:11:54.602368Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T121154485645Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T121154485645Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f733a5bed1432043103f900714b2b380780846b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T121154485645Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e896b7e1e676f48bab86d8c021e2c2b1c3417f829a14f5b1ee519f33e417b07 +size 5841425 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T121155334888Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T121155334888Z.json new file mode 100644 index 0000000000000000000000000000000000000000..936bb9628bb5f58e1941966268f73d5ad8753a72 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T121155334888Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888266752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123989592064, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888266752, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66888266752, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_peak": 8376399872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_peak": 136753184768, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:11:55.334409Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888266752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105835963904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888266752, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66888266752, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_peak": 8376399872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_peak": 136753184768, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:11:54.616672Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8376399872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136753184768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 752, + "segment_count_current_delta": 0, + "segment_count_peak": 752, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T121155334888Z.pickle", + "timestamp_utc": "2026-08-01T12:11:55.341586Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T121155334888Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T121155334888Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8536adacebed5a5a1edb9f444cefbe48dd1ef107 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T121155334888Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2456a3a6c379dffa14d89948845bd207bfff19c19f366a16a68ee906cc21abe6 +size 231524 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T121156051321Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T121156051321Z.json new file mode 100644 index 0000000000000000000000000000000000000000..90ebf0f35954720b3d2bef0654916afc30ffab48 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T121156051321Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888266752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123989592064, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888266752, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66888266752, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_peak": 8376399872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_peak": 136753184768, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:11:56.050897Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888266752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105835963904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888266752, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66888266752, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_peak": 8376399872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_peak": 136753184768, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:11:55.342565Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8376399872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136753184768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 752, + "segment_count_current_delta": 0, + "segment_count_peak": 752, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T121156051321Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.057882Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T121156051321Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T121156051321Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1dd4c62e778400b8855d33a123322726ed7203f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T121156051321Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49aea9909f30d32880744e712df18dad668e5416ee6b82e374db3b0cd7018578 +size 231524 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T121156765966Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T121156765966Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69a39c2addd3189d2ee33c6a0d10afa367cd17d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T121156765966Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888266752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123989592064, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888266752, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66888266752, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_peak": 8376399872, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_peak": 136753184768, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:11:56.765583Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888266752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105835963904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888266752, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66888266752, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_peak": 8376399872, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_peak": 136753184768, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:11:56.058820Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8376399872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136753184768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 752, + "segment_count_current_delta": 0, + "segment_count_peak": 752, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T121156765966Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.772008Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T121156765966Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T121156765966Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f866797a4a7a48267848bd429e53b9fb9685ddf5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T121156765966Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:330b8289345dde5be1d49a4919bfccc9aa95163164b891193023d9679dbfe8b1 +size 231524 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T121157480854Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T121157480854Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd0c8d3748a1b45bbbc3586cd94e18f57ea9eaa9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T121157480854Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888266752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123989592064, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888266752, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66888266752, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_peak": 8376399872, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_peak": 136753184768, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:11:57.480457Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888266752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105835963904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888266752, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66888266752, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_peak": 8376399872, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_peak": 136753184768, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:11:56.772953Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8376399872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136753184768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 752, + "segment_count_current_delta": 0, + "segment_count_peak": 752, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T121157480854Z.pickle", + "timestamp_utc": "2026-08-01T12:11:57.487025Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T121157480854Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T121157480854Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1dc780c2fc9fd17b9650178996706621a3dfcb2d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T121157480854Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:606d92a3b4065104b5a6f5371e9de32c928ec3b2f305cc644ecd4c9778dabb6f +size 231524 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T121158197112Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T121158197112Z.json new file mode 100644 index 0000000000000000000000000000000000000000..faf55d5b4eaca7759bfc29160fd65470c2c56120 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T121158197112Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888266752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123989592064, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888266752, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66888266752, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_peak": 8376399872, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_peak": 136753184768, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:11:58.196723Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888266752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105835963904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888266752, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66888266752, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_peak": 8376399872, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_peak": 136753184768, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:11:57.487976Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8376399872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136753184768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 752, + "segment_count_current_delta": 0, + "segment_count_peak": 752, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T121158197112Z.pickle", + "timestamp_utc": "2026-08-01T12:11:58.203879Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T121158197112Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T121158197112Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..be855a0f45b2c9b05c5f89e29b5dcf584f434ac3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T121158197112Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29f2345f16ba900b51f937333b761528fff654d79947d1ce55157c904e548ffb +size 231524 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T121247606898Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T121247606898Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8785197fc73f93069f0c3735d19f1f1392febf66 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T121247606898Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854632448, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 123989592064, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854632448, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66854632448, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 3014083584, + "inactive_split_bytes_peak": 8376399872, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 137189392384, + "reserved_bytes_peak": 137189392384, + "segment_count_current": 755, + "segment_count_peak": 755, + "timestamp_utc": "2026-08-01T12:12:47.606426Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854632448, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105835963904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888266752, + "active_bytes_peak": 123989592064, + "allocated_bytes_current": 66888266752, + "allocated_bytes_peak": 105835963904, + "device": 0, + "inactive_split_bytes_current": 5522197504, + "inactive_split_bytes_peak": 8376399872, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 136753184768, + "reserved_bytes_peak": 136753184768, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:11:58.204842Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3014083584, + "inactive_split_bytes_current_delta": -2508113920, + "inactive_split_bytes_peak": 8376399872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 137189392384, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137189392384, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 755, + "segment_count_current_delta": 3, + "segment_count_peak": 755, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T121247606898Z.pickle", + "timestamp_utc": "2026-08-01T12:12:47.691846Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T121247606898Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T121247606898Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cb1d64f0f3025f596abf2b874552d98ad4047d6e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T121247606898Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44eaacc3d3a48e0a70d6856a3b9ba2c9744b5437d805726bbf33a06a0547d073 +size 4023158 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T121149508191Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T121149508191Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a1598345706a2fde7d9181467b3d7eb0457c56a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T121149508191Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 31639626752, + "active_bytes_peak": 82194388480, + "active_bytes_peak_delta": 46728052224, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 82194388480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 82194388480, + "device": 1, + "inactive_split_bytes_current": 4914430976, + "inactive_split_bytes_peak": 7156222976, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:11:49.507529Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 31639626752, + "allocated_bytes_peak": 82194388480, + "allocated_bytes_peak_delta": 46728052224, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:10:41.423568Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4914430976, + "inactive_split_bytes_current_delta": 4781612032, + "inactive_split_bytes_peak": 7156222976, + "inactive_split_bytes_peak_delta": 6995836928, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 488, + "segment_count_current_delta": 417, + "segment_count_peak": 488, + "segment_count_peak_delta": 417, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T121149508191Z.pickle", + "timestamp_utc": "2026-08-01T12:11:49.617141Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T121149508191Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T121149508191Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cdaedc35bd88a5549a2279ea11ad4d782f48a296 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T121149508191Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d38c0bd032f441e9ff51ceb09d51c1ff073da6f5f2a1a5ce04d01e92f75bd566 +size 5373727 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T121150684145Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T121150684145Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d2b46ec51522c72e62f9e8011db9658826c3f30b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T121150684145Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105826068480, + "active_bytes_peak_delta": 23631680000, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 1, + "inactive_split_bytes_current": 4738270208, + "inactive_split_bytes_peak": 7156222976, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:50.683675Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105826068480, + "allocated_bytes_peak_delta": 23631680000, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 82194388480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 82194388480, + "device": 1, + "inactive_split_bytes_current": 4914430976, + "inactive_split_bytes_peak": 7156222976, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:11:49.634796Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4738270208, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 7156222976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107047026688, + "reserved_bytes_current_delta": 9558818816, + "reserved_bytes_peak": 107047026688, + "reserved_bytes_peak_delta": 9558818816, + "segment_count_current": 543, + "segment_count_current_delta": 55, + "segment_count_peak": 543, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T121150684145Z.pickle", + "timestamp_utc": "2026-08-01T12:11:50.791311Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T121150684145Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T121150684145Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..12805ee02b55e2947227e088a6c1314a21522dfd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T121150684145Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4669feca61feeaa42392deedb349fe7c9c99b3b57bbdd18b0cbe8b1d169db71 +size 5528373 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T121151890119Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T121151890119Z.json new file mode 100644 index 0000000000000000000000000000000000000000..40487e305f88a80de57f23ec33dbda78a996c7dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T121151890119Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105826068480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 1, + "inactive_split_bytes_current": 5851857920, + "inactive_split_bytes_peak": 7156222976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:51.889712Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105826068480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 1, + "inactive_split_bytes_current": 4738270208, + "inactive_split_bytes_peak": 7156222976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:50.808461Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5851857920, + "inactive_split_bytes_current_delta": 1113587712, + "inactive_split_bytes_peak": 7156222976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107047026688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107047026688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 543, + "segment_count_current_delta": 0, + "segment_count_peak": 543, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T121151890119Z.pickle", + "timestamp_utc": "2026-08-01T12:11:51.992975Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T121151890119Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T121151890119Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..38aecc78666a00d04bed60dcd67556cddf4bac07 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T121151890119Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e907dc722160571c31c3a6f7a36c840bccff20ba0c3a4d0bab7f5454ec19520 +size 5533835 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T121154485630Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T121154485630Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e9fe86c0121a1737711d1313f6a6e3f84138dec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T121154485630Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 18165348352, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9250763776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 136860139520, + "reserved_bytes_peak": 136860139520, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:54.485092Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 11589632, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 1, + "inactive_split_bytes_current": 5851857920, + "inactive_split_bytes_peak": 7156222976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:52.006072Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_current_delta": -109318144, + "inactive_split_bytes_peak": 9250763776, + "inactive_split_bytes_peak_delta": 2094540800, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 136860139520, + "reserved_bytes_current_delta": 29813112832, + "reserved_bytes_peak": 136860139520, + "reserved_bytes_peak_delta": 29813112832, + "segment_count_current": 747, + "segment_count_current_delta": 204, + "segment_count_peak": 747, + "segment_count_peak_delta": 204, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T121154485630Z.pickle", + "timestamp_utc": "2026-08-01T12:11:54.601550Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T121154485630Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T121154485630Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a1833839aae9a8554f263a0b9bfc72d613f23b0c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T121154485630Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68d5f40bb52121c83a04e4331598973892f8de0a5bddb1b46754469cc4f2b02a +size 5831677 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T121155334881Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T121155334881Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3428bded1fbaa9d5522c870bb91255b9b7cbd6d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T121155334881Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9250763776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 136860139520, + "reserved_bytes_peak": 136860139520, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:55.334399Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9250763776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 136860139520, + "reserved_bytes_peak": 136860139520, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:54.618283Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9250763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 136860139520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136860139520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T121155334881Z.pickle", + "timestamp_utc": "2026-08-01T12:11:55.341349Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T121155334881Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T121155334881Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2438a1aac7d7f80215e861daf0a15e4da2a28bba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T121155334881Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a2187ee0cb2624d7f4f628b1fd87a3b83c0a1fa196813e6aa599dd7e759e5fc +size 221790 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T121156051315Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T121156051315Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8eddf1be9e45a3065c0425a1440ed5d5ab7e8cd3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T121156051315Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9250763776, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 136860139520, + "reserved_bytes_peak": 136860139520, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:56.050899Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9250763776, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 136860139520, + "reserved_bytes_peak": 136860139520, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:55.342294Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9250763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 136860139520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136860139520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T121156051315Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.057566Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T121156051315Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T121156051315Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60bad1656ae67bff267927cd7e4d5a2daa60a325 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T121156051315Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fad5f73dc742595535a0963a7c501dce9595c5bf02c3ba2e111c99e0dd7f438 +size 221790 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T121156765954Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T121156765954Z.json new file mode 100644 index 0000000000000000000000000000000000000000..749117947a5f0a16cf1bf207c4f3d9399d618206 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T121156765954Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9250763776, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 136860139520, + "reserved_bytes_peak": 136860139520, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:56.765562Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9250763776, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 136860139520, + "reserved_bytes_peak": 136860139520, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:56.058503Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9250763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 136860139520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136860139520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T121156765954Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.771895Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T121156765954Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T121156765954Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a7e71c26471a77296ee6fe79843e8d4988cd666 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T121156765954Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72ba0fa273eef66b0c91f185acc0e3c47f0350f9c3c1a61037685d364df71a78 +size 221790 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T121157480850Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T121157480850Z.json new file mode 100644 index 0000000000000000000000000000000000000000..85d8f09872b018a3bd23c401598f5dba0b00a1d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T121157480850Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9250763776, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 136860139520, + "reserved_bytes_peak": 136860139520, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:57.480458Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9250763776, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 136860139520, + "reserved_bytes_peak": 136860139520, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:56.772876Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9250763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 136860139520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136860139520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T121157480850Z.pickle", + "timestamp_utc": "2026-08-01T12:11:57.486824Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T121157480850Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T121157480850Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..96079a873f57147604100cf622f9c60adfa76bc1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T121157480850Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b531bdd0ea36c684c936a7731f9084f320135eb596ac90567384c8e76ba6c2c2 +size 221790 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T121158197016Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T121158197016Z.json new file mode 100644 index 0000000000000000000000000000000000000000..694eb28ad4b96d058471de12b562ff59ab9f6252 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T121158197016Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9250763776, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 136860139520, + "reserved_bytes_peak": 136860139520, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:58.196622Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9250763776, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 136860139520, + "reserved_bytes_peak": 136860139520, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:57.487754Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9250763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 136860139520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136860139520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T121158197016Z.pickle", + "timestamp_utc": "2026-08-01T12:11:58.202769Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T121158197016Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T121158197016Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5587c48def952a937e9af5e8def64ab4be3209a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T121158197016Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36d3e8eb466e2ecee43bcccfbd5cd2bb244d6219ce032d44e39cd8b214a73892 +size 221790 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T121247606902Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T121247606902Z.json new file mode 100644 index 0000000000000000000000000000000000000000..df42d18e587f5ed7d58383be225da8f1759f304b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T121247606902Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854491136, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854491136, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66854491136, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 2970184704, + "inactive_split_bytes_peak": 9250763776, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 137296347136, + "reserved_bytes_peak": 137296347136, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:12:47.606429Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854491136, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 1, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9250763776, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 136860139520, + "reserved_bytes_peak": 136860139520, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:58.203693Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2970184704, + "inactive_split_bytes_current_delta": -2772355072, + "inactive_split_bytes_peak": 9250763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 137296347136, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137296347136, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 750, + "segment_count_current_delta": 3, + "segment_count_peak": 750, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T121247606902Z.pickle", + "timestamp_utc": "2026-08-01T12:12:47.695321Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T121247606902Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T121247606902Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..32d5c49e20c3e561965e87bd5971132f2725b1b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T121247606902Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af8186be59180bbe00686988851577a8ba440aa62a234155eb11c998e14c90ec +size 4014404 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T121149508256Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T121149508256Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a8f900faff9661144f50a8dede070497a79f387c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T121149508256Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 31639626752, + "active_bytes_peak": 82194388480, + "active_bytes_peak_delta": 46728052224, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 82194388480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 82194388480, + "device": 2, + "inactive_split_bytes_current": 4916528128, + "inactive_split_bytes_peak": 7158402048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:11:49.507574Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 31639626752, + "allocated_bytes_peak": 82194388480, + "allocated_bytes_peak_delta": 46728052224, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:10:41.423847Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4916528128, + "inactive_split_bytes_current_delta": 4783709184, + "inactive_split_bytes_peak": 7158402048, + "inactive_split_bytes_peak_delta": 6998016000, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 488, + "segment_count_current_delta": 417, + "segment_count_peak": 488, + "segment_count_peak_delta": 417, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T121149508256Z.pickle", + "timestamp_utc": "2026-08-01T12:11:49.617680Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T121149508256Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T121149508256Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..88e0aea864b5c0fba93ea3356ed8124ad97d7748 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T121149508256Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8758f1d9a010a7cf2056ef7236ec45c6529ee8fb43d00db127a92ada6e3b4c7e +size 5373570 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T121150684125Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T121150684125Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed15b18d68054dbeb479863b60cfc64abf7b1eee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T121150684125Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105826068480, + "active_bytes_peak_delta": 23631680000, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 2, + "inactive_split_bytes_current": 4738270208, + "inactive_split_bytes_peak": 7158402048, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:50.683684Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105826068480, + "allocated_bytes_peak_delta": 23631680000, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 82194388480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 82194388480, + "device": 2, + "inactive_split_bytes_current": 4916528128, + "inactive_split_bytes_peak": 7158402048, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:11:49.635342Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4738270208, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 7158402048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107047026688, + "reserved_bytes_current_delta": 9558818816, + "reserved_bytes_peak": 107047026688, + "reserved_bytes_peak_delta": 9558818816, + "segment_count_current": 543, + "segment_count_current_delta": 55, + "segment_count_peak": 543, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T121150684125Z.pickle", + "timestamp_utc": "2026-08-01T12:11:50.791555Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T121150684125Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T121150684125Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ec464fbf814dd3843ffe435c4117ae8928f0d96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T121150684125Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5db2a1233ccb21556f4611c61a5b5c82553eb9469aef0395269e7ff56d05c2cb +size 5525147 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T121151890148Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T121151890148Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6317c52cb54076d3a73bb70fed7e030f776a2ef9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T121151890148Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105826068480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 2, + "inactive_split_bytes_current": 5849760768, + "inactive_split_bytes_peak": 7158402048, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:51.889742Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105826068480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 2, + "inactive_split_bytes_current": 4738270208, + "inactive_split_bytes_peak": 7158402048, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:50.805153Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5849760768, + "inactive_split_bytes_current_delta": 1111490560, + "inactive_split_bytes_peak": 7158402048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107047026688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107047026688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 543, + "segment_count_current_delta": 0, + "segment_count_peak": 543, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T121151890148Z.pickle", + "timestamp_utc": "2026-08-01T12:11:51.995374Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T121151890148Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T121151890148Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c854d719f0f02bc2ba044b0677f52cb47885d1f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T121151890148Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:548c57cb7f7218305140dcf1ffa43f33cad6c0a8578438821a5121784a79cd82 +size 5543094 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T121154485669Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T121154485669Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2c008f258adbcfc47f7507dbcdfeccff297a6f28 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T121154485669Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 18165348352, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9254958080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 136858042368, + "reserved_bytes_peak": 136858042368, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:11:54.485126Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 11589632, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 2, + "inactive_split_bytes_current": 5849760768, + "inactive_split_bytes_peak": 7158402048, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:52.008320Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_current_delta": -107220992, + "inactive_split_bytes_peak": 9254958080, + "inactive_split_bytes_peak_delta": 2096556032, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 136858042368, + "reserved_bytes_current_delta": 29811015680, + "reserved_bytes_peak": 136858042368, + "reserved_bytes_peak_delta": 29811015680, + "segment_count_current": 746, + "segment_count_current_delta": 203, + "segment_count_peak": 746, + "segment_count_peak_delta": 203, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T121154485669Z.pickle", + "timestamp_utc": "2026-08-01T12:11:54.603778Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T121154485669Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T121154485669Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d8ec2f1f25cd61c0cfa717104143ef08748010de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T121154485669Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab8657dec31e1eab70d7029b5f4df63e7e788890e05779772be1de1d4f47acbe +size 5978276 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T121155334890Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T121155334890Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70059fc1296a59719340cc7579745116235f0759 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T121155334890Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9254958080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 136858042368, + "reserved_bytes_peak": 136858042368, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:11:55.334429Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9254958080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 136858042368, + "reserved_bytes_peak": 136858042368, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:11:54.618258Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9254958080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 136858042368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136858042368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 746, + "segment_count_current_delta": 0, + "segment_count_peak": 746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T121155334890Z.pickle", + "timestamp_utc": "2026-08-01T12:11:55.341425Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T121155334890Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T121155334890Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ae6c4c035faf1d9dfc5746e0941a38d86d6ee8d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T121155334890Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e383238f6a22165a168b5652af91761451a58ae18c1522466b2b1d22651f248 +size 227313 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T121156051368Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T121156051368Z.json new file mode 100644 index 0000000000000000000000000000000000000000..10f615222ac4aa79d5ce52b799482fb5d524ba32 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T121156051368Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9254958080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 136858042368, + "reserved_bytes_peak": 136858042368, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:11:56.050934Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9254958080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 136858042368, + "reserved_bytes_peak": 136858042368, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:11:55.342386Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9254958080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 136858042368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136858042368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 746, + "segment_count_current_delta": 0, + "segment_count_peak": 746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T121156051368Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.057784Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T121156051368Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T121156051368Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4bffef72751a1e0be27c6e4f7f826369dda36b4c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T121156051368Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26ada138c54f117e10025cc4fe92e3be14898cbb3314e9e8a70a1eda5c319afa +size 227313 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T121156765990Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T121156765990Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3047cf3717c840319d5bf0ddb75d5e505b2c6b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T121156765990Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9254958080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 136858042368, + "reserved_bytes_peak": 136858042368, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:11:56.765594Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9254958080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 136858042368, + "reserved_bytes_peak": 136858042368, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:11:56.058723Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9254958080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 136858042368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136858042368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 746, + "segment_count_current_delta": 0, + "segment_count_peak": 746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T121156765990Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.771958Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T121156765990Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T121156765990Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d31185736bf7d571fbb79f6faac6631e222a9c87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T121156765990Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e75bc37a1112eaa35d124200a3d8019b093f4f9b68f49a142b0c5e76092998cb +size 227313 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T121157480902Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T121157480902Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf711a13e64ed3ec478b816e22a2ed4f9200bf18 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T121157480902Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9254958080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 136858042368, + "reserved_bytes_peak": 136858042368, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:11:57.480495Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9254958080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 136858042368, + "reserved_bytes_peak": 136858042368, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:11:56.772910Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9254958080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 136858042368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136858042368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 746, + "segment_count_current_delta": 0, + "segment_count_peak": 746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T121157480902Z.pickle", + "timestamp_utc": "2026-08-01T12:11:57.486889Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T121157480902Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T121157480902Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ebe591d90c1c0515e027dd825cad993dcfd53605 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T121157480902Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ed88e4688c118b40448c59496ed6781e570c3ea595a349593f73dee954995d6 +size 227313 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T121158197052Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T121158197052Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4c7f750a50b90ba5e2903d05817c51b00e7a14e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T121158197052Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9254958080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 136858042368, + "reserved_bytes_peak": 136858042368, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:11:58.196664Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9254958080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 136858042368, + "reserved_bytes_peak": 136858042368, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:11:57.487814Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9254958080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 136858042368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136858042368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 746, + "segment_count_current_delta": 0, + "segment_count_peak": 746, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T121158197052Z.pickle", + "timestamp_utc": "2026-08-01T12:11:58.202869Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T121158197052Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T121158197052Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..243619f5b3caadc68875cda505f5a2af956540e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T121158197052Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e75e178f998ae3b74ffb170e2dac6b6c81d5366138298a4f7aadc9f9e3b85b4 +size 227313 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T121247606919Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T121247606919Z.json new file mode 100644 index 0000000000000000000000000000000000000000..181503989216f0a3365492b229bcda82816bbf4b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T121247606919Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854491136, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854491136, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66854491136, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 2970184704, + "inactive_split_bytes_peak": 9254958080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 137294249984, + "reserved_bytes_peak": 137294249984, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:12:47.606427Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854491136, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 2, + "inactive_split_bytes_current": 5742539776, + "inactive_split_bytes_peak": 9254958080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 136858042368, + "reserved_bytes_peak": 136858042368, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:11:58.203790Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2970184704, + "inactive_split_bytes_current_delta": -2772355072, + "inactive_split_bytes_peak": 9254958080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 137294249984, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137294249984, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 749, + "segment_count_current_delta": 3, + "segment_count_peak": 749, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T121247606919Z.pickle", + "timestamp_utc": "2026-08-01T12:12:47.697553Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T121247606919Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T121247606919Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..66e00d7dee1c7c7e7d529786af462f972e8ff6be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T121247606919Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38d41df3ed6f8f2f310eabd8cf5217f0b6e9aa1d3cdab8e6b997e776872a829e +size 4013402 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T121149508269Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T121149508269Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eaa11deea349ccf96d6435ab8e78dba4a99ad7ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T121149508269Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 31639626752, + "active_bytes_peak": 82194388480, + "active_bytes_peak_delta": 46728052224, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 82194388480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 82194388480, + "device": 3, + "inactive_split_bytes_current": 4912333824, + "inactive_split_bytes_peak": 7154371584, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:11:49.507588Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 31639626752, + "allocated_bytes_peak": 82194388480, + "allocated_bytes_peak_delta": 46728052224, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:10:41.423739Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4912333824, + "inactive_split_bytes_current_delta": 4779514880, + "inactive_split_bytes_peak": 7154371584, + "inactive_split_bytes_peak_delta": 6993985536, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 488, + "segment_count_current_delta": 417, + "segment_count_peak": 488, + "segment_count_peak_delta": 417, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T121149508269Z.pickle", + "timestamp_utc": "2026-08-01T12:11:49.623675Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T121149508269Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T121149508269Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f0fb021f5864486f4ffc64f2311ed698ca724b25 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T121149508269Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eab5b750845a5930f62de5c9bc39dd5dc9d9080af8de44c28798636b51514267 +size 5373808 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T121150684060Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T121150684060Z.json new file mode 100644 index 0000000000000000000000000000000000000000..450f1f3a97f4cbc22c2cce8733f3decfc18beb4e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T121150684060Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105826068480, + "active_bytes_peak_delta": 23631680000, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 3, + "inactive_split_bytes_current": 4734075904, + "inactive_split_bytes_peak": 7154371584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:50.683635Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105826068480, + "allocated_bytes_peak_delta": 23631680000, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 82194388480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 82194388480, + "device": 3, + "inactive_split_bytes_current": 4912333824, + "inactive_split_bytes_peak": 7154371584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:11:49.640365Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4734075904, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 7154371584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107047026688, + "reserved_bytes_current_delta": 9558818816, + "reserved_bytes_peak": 107047026688, + "reserved_bytes_peak_delta": 9558818816, + "segment_count_current": 543, + "segment_count_current_delta": 55, + "segment_count_peak": 543, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T121150684060Z.pickle", + "timestamp_utc": "2026-08-01T12:11:50.787195Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T121150684060Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T121150684060Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..932c9e11fa19463ecae3c9c4dc72bbaeb402fc20 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T121150684060Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02d9f19b355f5d1b0f23735bb38fc18fa156ec66df7041c9c1de02131c89407a +size 5565767 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T121151890119Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T121151890119Z.json new file mode 100644 index 0000000000000000000000000000000000000000..63049610be6dba2946b61f323cbcf064ed4ee56f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T121151890119Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105826068480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 3, + "inactive_split_bytes_current": 5650531328, + "inactive_split_bytes_peak": 7154371584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:51.889694Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105826068480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 3, + "inactive_split_bytes_current": 4734075904, + "inactive_split_bytes_peak": 7154371584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:50.799208Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5650531328, + "inactive_split_bytes_current_delta": 916455424, + "inactive_split_bytes_peak": 7154371584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107047026688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107047026688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 543, + "segment_count_current_delta": 0, + "segment_count_peak": 543, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T121151890119Z.pickle", + "timestamp_utc": "2026-08-01T12:11:51.995625Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T121151890119Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T121151890119Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dc51c243dee9da00b1d500d7f03132db13d3500f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T121151890119Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55e1f9ee650332e5c32292f34f76fef01b7f29a4abdcf3a7086f7df7730a0ed8 +size 5533078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T121154485636Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T121154485636Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1515faa8df114c46738f9b7adc93e7b1beac2193 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T121154485636Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123988533248, + "active_bytes_peak_delta": 18162464768, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 8852904960, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 136658812928, + "reserved_bytes_peak": 136658812928, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:54.485136Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105834774528, + "allocated_bytes_peak_delta": 8706048, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 3, + "inactive_split_bytes_current": 5650531328, + "inactive_split_bytes_peak": 7154371584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:52.007574Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_current_delta": 89911296, + "inactive_split_bytes_peak": 8852904960, + "inactive_split_bytes_peak_delta": 1698533376, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 136658812928, + "reserved_bytes_current_delta": 29611786240, + "reserved_bytes_peak": 136658812928, + "reserved_bytes_peak_delta": 29611786240, + "segment_count_current": 747, + "segment_count_current_delta": 204, + "segment_count_peak": 747, + "segment_count_peak_delta": 204, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T121154485636Z.pickle", + "timestamp_utc": "2026-08-01T12:11:54.605624Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T121154485636Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T121154485636Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff51f45099fee257ec3831aed01f722a4ed9621d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T121154485636Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b27698579bf4f4ad21af863f4ab036ba8b74a99936c747d9c939a7c9d500347 +size 5831341 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T121155334828Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T121155334828Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea3f770aa5fcd22e4e94fbba570b24b4b05a0078 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T121155334828Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123988533248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 8852904960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 136658812928, + "reserved_bytes_peak": 136658812928, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:55.334400Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834774528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 8852904960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 136658812928, + "reserved_bytes_peak": 136658812928, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:54.621283Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8852904960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 136658812928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136658812928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T121155334828Z.pickle", + "timestamp_utc": "2026-08-01T12:11:55.340777Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T121155334828Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T121155334828Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..030b29dc1ada2a057114f4fd00b792e3561e7754 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T121155334828Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2aa74e73f75bc7225471e6bbbd03492bed432f349bb06474647ea948baf824ea +size 221588 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T121156051305Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T121156051305Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9be60da126e7e07aef3358cd55a6b5a1d6022f68 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T121156051305Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123988533248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 5738345472, + "inactive_split_bytes_peak": 8852904960, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 136658812928, + "reserved_bytes_peak": 136658812928, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:56.050896Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834774528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 8852904960, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 136658812928, + "reserved_bytes_peak": 136658812928, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:55.341726Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5738345472, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8852904960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 136658812928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136658812928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T121156051305Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.057359Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T121156051305Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T121156051305Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f74e144446bb7cc7b3fa77e99315166b77c2105 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T121156051305Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f07037aa832097bb66835512908c49e1ba7fd95d4081bbd246857af3cfe792e8 +size 221616 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T121156765940Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T121156765940Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1beb2894396ae26687dab031c85840b1f19f171c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T121156765940Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123988533248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 5738345472, + "inactive_split_bytes_peak": 8852904960, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 136658812928, + "reserved_bytes_peak": 136658812928, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:56.765555Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834774528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 5738345472, + "inactive_split_bytes_peak": 8852904960, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 136658812928, + "reserved_bytes_peak": 136658812928, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:56.058283Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5738345472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8852904960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 136658812928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136658812928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T121156765940Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.771863Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T121156765940Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T121156765940Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9554e415462fc0d3a4c56eee9e60ca813f7b41c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T121156765940Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2a5c45227f3ccae0566ea4b3707b07328c06162165f93766fb6b15e917c9479 +size 221616 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T121157480853Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T121157480853Z.json new file mode 100644 index 0000000000000000000000000000000000000000..50966bafeaf5e4910667efe6e9925cb88cfffd55 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T121157480853Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123988533248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 5738345472, + "inactive_split_bytes_peak": 8852904960, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 136658812928, + "reserved_bytes_peak": 136658812928, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:57.480454Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834774528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 5738345472, + "inactive_split_bytes_peak": 8852904960, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 136658812928, + "reserved_bytes_peak": 136658812928, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:56.772814Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5738345472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8852904960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 136658812928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136658812928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T121157480853Z.pickle", + "timestamp_utc": "2026-08-01T12:11:57.486709Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T121157480853Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T121157480853Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f17b65b7de249f3f880c5967e7442b8431d332da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T121157480853Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63e813bb06e6afe61524310996a101f41358dec0883047181676069c297a67e4 +size 221616 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T121158197017Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T121158197017Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c0bf29db91f429019475ead3ab3d98b396f7dff4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T121158197017Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123988533248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 5738345472, + "inactive_split_bytes_peak": 8852904960, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 136658812928, + "reserved_bytes_peak": 136658812928, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:58.196638Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834774528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 5738345472, + "inactive_split_bytes_peak": 8852904960, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 136658812928, + "reserved_bytes_peak": 136658812928, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:57.487625Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5738345472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8852904960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 136658812928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136658812928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T121158197017Z.pickle", + "timestamp_utc": "2026-08-01T12:11:58.202819Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T121158197017Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T121158197017Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d9c2ad9b98f5135e9f89e0e768e46004e52eb36e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T121158197017Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c04f60c96834c07e5abf666c97a08797d13e8980a146f10cbb4495daf4bfe156 +size 221616 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T121247606897Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T121247606897Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e22277ea8f4b91cd68ef79eed7bef48bb07e81b4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T121247606897Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854491136, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 123988533248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854491136, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66854491136, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 2972281856, + "inactive_split_bytes_peak": 8852904960, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 137095020544, + "reserved_bytes_peak": 137095020544, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:12:47.606417Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854491136, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105834774528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123988533248, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105834774528, + "device": 3, + "inactive_split_bytes_current": 5738345472, + "inactive_split_bytes_peak": 8852904960, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 136658812928, + "reserved_bytes_peak": 136658812928, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:58.203726Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2972281856, + "inactive_split_bytes_current_delta": -2766063616, + "inactive_split_bytes_peak": 8852904960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 137095020544, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137095020544, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 750, + "segment_count_current_delta": 3, + "segment_count_peak": 750, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T121247606897Z.pickle", + "timestamp_utc": "2026-08-01T12:12:47.696916Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T121247606897Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T121247606897Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..59638bd34dc7b69e990f44f18c0ef384d29d4309 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T121247606897Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:678828c9e05132e607b14477f8596a497f1921b772123a1551313eda879a1373 +size 4013922 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T121149508085Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T121149508085Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eb0f7eb08b90a6ba22fdc76ac04fe9c818065143 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T121149508085Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871823360, + "active_bytes_current_delta": 31639925760, + "active_bytes_peak": 82188125696, + "active_bytes_peak_delta": 46721347072, + "after": { + "active_bytes_current": 66871823360, + "active_bytes_peak": 82188125696, + "allocated_bytes_current": 66871823360, + "allocated_bytes_peak": 82188125696, + "device": 4, + "inactive_split_bytes_current": 5368771584, + "inactive_split_bytes_peak": 7296868352, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T12:11:49.507604Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871823360, + "allocated_bytes_current_delta": 31639925760, + "allocated_bytes_peak": 82188125696, + "allocated_bytes_peak_delta": 46721347072, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:10:41.423166Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5368771584, + "inactive_split_bytes_current_delta": 5236395008, + "inactive_split_bytes_peak": 7296868352, + "inactive_split_bytes_peak_delta": 7143458816, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 490, + "segment_count_current_delta": 419, + "segment_count_peak": 490, + "segment_count_peak_delta": 419, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T121149508085Z.pickle", + "timestamp_utc": "2026-08-01T12:11:49.625122Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T121149508085Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T121149508085Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0cbb795438b1b59c6d524bd2b286a8d5f9294cba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T121149508085Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e92358a83f16b080e1ae3c2cac3e168d6a7d9ef5cbec3e68926cc1e2fed1f76 +size 5385105 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T121150684047Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T121150684047Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0aad63f132e3166db07ab38e0b2545acf18feaf5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T121150684047Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871823360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105824616448, + "active_bytes_peak_delta": 23636490752, + "after": { + "active_bytes_current": 66871823360, + "active_bytes_peak": 105824616448, + "allocated_bytes_current": 66871823360, + "allocated_bytes_peak": 105824616448, + "device": 4, + "inactive_split_bytes_current": 5196805120, + "inactive_split_bytes_peak": 7296868352, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:11:50.683635Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871823360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105824616448, + "allocated_bytes_peak_delta": 23636490752, + "before": { + "active_bytes_current": 66871823360, + "active_bytes_peak": 82188125696, + "allocated_bytes_current": 66871823360, + "allocated_bytes_peak": 82188125696, + "device": 4, + "inactive_split_bytes_current": 5368771584, + "inactive_split_bytes_peak": 7296868352, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T12:11:49.643231Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5196805120, + "inactive_split_bytes_current_delta": -171966464, + "inactive_split_bytes_peak": 7296868352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107047026688, + "reserved_bytes_current_delta": 9558818816, + "reserved_bytes_peak": 107047026688, + "reserved_bytes_peak_delta": 9558818816, + "segment_count_current": 545, + "segment_count_current_delta": 55, + "segment_count_peak": 545, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T121150684047Z.pickle", + "timestamp_utc": "2026-08-01T12:11:50.792078Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T121150684047Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T121150684047Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..88bb2c1b20bb3b499859547e8ad856afd512f2ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T121150684047Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1e361967f161bb7b82fae2f2bcdf9854c48d2a59b1c2a7ef76fe8728e3caa0f +size 5541957 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T121151890171Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T121151890171Z.json new file mode 100644 index 0000000000000000000000000000000000000000..78496371e755c6932609078d9f7973fb01b5539d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T121151890171Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871823360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105825403392, + "active_bytes_peak_delta": 786944, + "after": { + "active_bytes_current": 66871823360, + "active_bytes_peak": 105825403392, + "allocated_bytes_current": 66871823360, + "allocated_bytes_peak": 105825403392, + "device": 4, + "inactive_split_bytes_current": 6192952320, + "inactive_split_bytes_peak": 7296868352, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:11:51.889700Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871823360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105825403392, + "allocated_bytes_peak_delta": 786944, + "before": { + "active_bytes_current": 66871823360, + "active_bytes_peak": 105824616448, + "allocated_bytes_current": 66871823360, + "allocated_bytes_peak": 105824616448, + "device": 4, + "inactive_split_bytes_current": 5196805120, + "inactive_split_bytes_peak": 7296868352, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:11:50.805689Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6192952320, + "inactive_split_bytes_current_delta": 996147200, + "inactive_split_bytes_peak": 7296868352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107047026688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107047026688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T121151890171Z.pickle", + "timestamp_utc": "2026-08-01T12:11:51.995580Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T121151890171Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T121151890171Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..addcac2c6c6d4f5739dbd7175c2e97d38fdcf30d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T121151890171Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfc82d1028d66291621d357f5b5c5c632054bab525e68c68dc18b4b64080d612 +size 5540778 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T121154485634Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T121154485634Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b5d4dc68ddd785c28ee379caf2d21c6ada6740e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T121154485634Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888866816, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123980492800, + "active_bytes_peak_delta": 18155089408, + "after": { + "active_bytes_current": 66888866816, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66888866816, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_peak": 9139266560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137495576576, + "reserved_bytes_peak": 137495576576, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:11:54.485088Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888866816, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105835252736, + "allocated_bytes_peak_delta": 9849344, + "before": { + "active_bytes_current": 66871823360, + "active_bytes_peak": 105825403392, + "allocated_bytes_current": 66871823360, + "allocated_bytes_peak": 105825403392, + "device": 4, + "inactive_split_bytes_current": 6192952320, + "inactive_split_bytes_peak": 7296868352, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:11:52.008652Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_current_delta": -329519104, + "inactive_split_bytes_peak": 9139266560, + "inactive_split_bytes_peak_delta": 1842398208, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137495576576, + "reserved_bytes_current_delta": 30448549888, + "reserved_bytes_peak": 137495576576, + "reserved_bytes_peak_delta": 30448549888, + "segment_count_current": 750, + "segment_count_current_delta": 205, + "segment_count_peak": 750, + "segment_count_peak_delta": 205, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T121154485634Z.pickle", + "timestamp_utc": "2026-08-01T12:11:54.603050Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T121154485634Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T121154485634Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..265e7290bf5734a0cae7938bd96d0fc8895d5ef3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T121154485634Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4557251be12b343986fa3e7c04318a7b12dde692378954b03953f69a0697b708 +size 5959144 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T121155334890Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T121155334890Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e9af679adbbf0b9a95576fb140c495423a4d5864 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T121155334890Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888866816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123980492800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888866816, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66888866816, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_peak": 9139266560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137495576576, + "reserved_bytes_peak": 137495576576, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:11:55.334418Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888866816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105835252736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888866816, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66888866816, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_peak": 9139266560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137495576576, + "reserved_bytes_peak": 137495576576, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:11:54.617325Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9139266560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137495576576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137495576576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 750, + "segment_count_current_delta": 0, + "segment_count_peak": 750, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T121155334890Z.pickle", + "timestamp_utc": "2026-08-01T12:11:55.341560Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T121155334890Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T121155334890Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..668a4adebb93dfef0958827cfea3816ec575a2b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T121155334890Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6991475ff8b42491954bcfde8e01f00c9bb2699264968878740ba1a515844026 +size 235608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T121156051339Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T121156051339Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4cb0a91cd56553201d855d41f4074b9cccf39c18 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T121156051339Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888866816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123980492800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888866816, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66888866816, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_peak": 9139266560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137495576576, + "reserved_bytes_peak": 137495576576, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:11:56.050922Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888866816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105835252736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888866816, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66888866816, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_peak": 9139266560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137495576576, + "reserved_bytes_peak": 137495576576, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:11:55.342531Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9139266560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137495576576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137495576576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 750, + "segment_count_current_delta": 0, + "segment_count_peak": 750, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T121156051339Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.057652Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T121156051339Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T121156051339Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..12111ec6b2c576745fb5de7ce8d00f8f50c7b3d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T121156051339Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1790b3afca7126d9258904fd71d2a091d2f1f0ce7b5226419f25317c180b56cc +size 235608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T121156765978Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T121156765978Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5783f5a7b90b2f93dfc310cef67e504404b24851 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T121156765978Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888866816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123980492800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888866816, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66888866816, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_peak": 9139266560, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137495576576, + "reserved_bytes_peak": 137495576576, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:11:56.765575Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888866816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105835252736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888866816, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66888866816, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_peak": 9139266560, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137495576576, + "reserved_bytes_peak": 137495576576, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:11:56.058605Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9139266560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137495576576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137495576576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 750, + "segment_count_current_delta": 0, + "segment_count_peak": 750, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T121156765978Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.772111Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T121156765978Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T121156765978Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9de229d3d937991bab47b8dc423855b975ab55d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T121156765978Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97de777ab84bfb8c552f37dc3df5b14288c3dea832450dd96698fd9f2a9d1d23 +size 235608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T121157480865Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T121157480865Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7d412fc3284c59ab60d9590d7a27a0c1eecab968 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T121157480865Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888866816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123980492800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888866816, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66888866816, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_peak": 9139266560, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137495576576, + "reserved_bytes_peak": 137495576576, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:11:57.480471Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888866816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105835252736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888866816, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66888866816, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_peak": 9139266560, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137495576576, + "reserved_bytes_peak": 137495576576, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:11:56.773058Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9139266560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137495576576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137495576576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 750, + "segment_count_current_delta": 0, + "segment_count_peak": 750, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T121157480865Z.pickle", + "timestamp_utc": "2026-08-01T12:11:57.486976Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T121157480865Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T121157480865Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0b40439cbbad6f148bc0fb88bd55bdf1b4bf1f13 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T121157480865Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:249c8521b32e2102cc4ece5a687f0f1adbc1e677b3b5d6583af4b1a968a77920 +size 235608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T121158197123Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T121158197123Z.json new file mode 100644 index 0000000000000000000000000000000000000000..30a579cb88e49d278f88d9ecc124ced2695898a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T121158197123Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888866816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123980492800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888866816, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66888866816, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_peak": 9139266560, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137495576576, + "reserved_bytes_peak": 137495576576, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:11:58.196728Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888866816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105835252736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888866816, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66888866816, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_peak": 9139266560, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137495576576, + "reserved_bytes_peak": 137495576576, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:11:57.487934Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9139266560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137495576576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137495576576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 750, + "segment_count_current_delta": 0, + "segment_count_peak": 750, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T121158197123Z.pickle", + "timestamp_utc": "2026-08-01T12:11:58.203869Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T121158197123Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T121158197123Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b7424e1053b28f3004bfbe10056a6957f6c64f78 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T121158197123Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc282a793325800f848b5e93095e1088350535cfdb2ba728386cead9cc1a4c5e +size 235608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T121247606908Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T121247606908Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0131f01b0b8a9115203d235ca39966b3d539bfce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T121247606908Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855232512, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 123980492800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855232512, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66855232512, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 3325959168, + "inactive_split_bytes_peak": 9139266560, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137931784192, + "reserved_bytes_peak": 137931784192, + "segment_count_current": 753, + "segment_count_peak": 753, + "timestamp_utc": "2026-08-01T12:12:47.606416Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855232512, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105835252736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888866816, + "active_bytes_peak": 123980492800, + "allocated_bytes_current": 66888866816, + "allocated_bytes_peak": 105835252736, + "device": 4, + "inactive_split_bytes_current": 5863433216, + "inactive_split_bytes_peak": 9139266560, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137495576576, + "reserved_bytes_peak": 137495576576, + "segment_count_current": 750, + "segment_count_peak": 750, + "timestamp_utc": "2026-08-01T12:11:58.204814Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3325959168, + "inactive_split_bytes_current_delta": -2537474048, + "inactive_split_bytes_peak": 9139266560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137931784192, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137931784192, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 753, + "segment_count_current_delta": 3, + "segment_count_peak": 753, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T121247606908Z.pickle", + "timestamp_utc": "2026-08-01T12:12:47.691312Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T121247606908Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T121247606908Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e6d1657fc92b0d6c7911c78d447703425d885a8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T121247606908Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23fcd34f161e059b85976f34dae4fa68c89ef9c2af3a61bcead4ac124463884d +size 4023710 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T121149508227Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T121149508227Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0246896beb497325cdf4350090fbbfb15928a3d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T121149508227Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 31639626752, + "active_bytes_peak": 82194388480, + "active_bytes_peak_delta": 46728052224, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 82194388480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 82194388480, + "device": 5, + "inactive_split_bytes_current": 4916528128, + "inactive_split_bytes_peak": 7158320128, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:11:49.507544Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 31639626752, + "allocated_bytes_peak": 82194388480, + "allocated_bytes_peak_delta": 46728052224, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:10:41.423595Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4916528128, + "inactive_split_bytes_current_delta": 4783709184, + "inactive_split_bytes_peak": 7158320128, + "inactive_split_bytes_peak_delta": 6997934080, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 488, + "segment_count_current_delta": 417, + "segment_count_peak": 488, + "segment_count_peak_delta": 417, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T121149508227Z.pickle", + "timestamp_utc": "2026-08-01T12:11:49.617670Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T121149508227Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T121149508227Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9905f400c22c180dc4ef04f81a2ebb85a0285078 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T121149508227Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c96fcee5821520c72592725e27155febb72c3b1ed7abf055ba10f4e56326e6ae +size 5373390 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T121150684030Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T121150684030Z.json new file mode 100644 index 0000000000000000000000000000000000000000..43f45354040dd96e0d9c3f57b6614c2a0ca69a60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T121150684030Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105828186624, + "active_bytes_peak_delta": 23633798144, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105828186624, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105828186624, + "device": 5, + "inactive_split_bytes_current": 5495342080, + "inactive_split_bytes_peak": 7158320128, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107017666560, + "reserved_bytes_peak": 107017666560, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:11:50.683604Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105828186624, + "allocated_bytes_peak_delta": 23633798144, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 82194388480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 82194388480, + "device": 5, + "inactive_split_bytes_current": 4916528128, + "inactive_split_bytes_peak": 7158320128, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:11:49.637779Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5495342080, + "inactive_split_bytes_current_delta": 578813952, + "inactive_split_bytes_peak": 7158320128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107017666560, + "reserved_bytes_current_delta": 9529458688, + "reserved_bytes_peak": 107017666560, + "reserved_bytes_peak_delta": 9529458688, + "segment_count_current": 545, + "segment_count_current_delta": 57, + "segment_count_peak": 545, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T121150684030Z.pickle", + "timestamp_utc": "2026-08-01T12:11:50.789280Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T121150684030Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T121150684030Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c5e776cab73377d276f68f3dd1dd5161c20a1d45 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T121150684030Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4646d7c9dfa7ce61dd454e557c7b043a153cf4898699e34b1ae8d0c912d97f5 +size 5525653 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T121151890096Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T121151890096Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f4282fb0eb9c3af825f07eb8a85898f12743588 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T121151890096Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105828186624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105828186624, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105828186624, + "device": 5, + "inactive_split_bytes_current": 6011241472, + "inactive_split_bytes_peak": 7158320128, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107487428608, + "reserved_bytes_peak": 107487428608, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T12:11:51.889669Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105828186624, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105828186624, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105828186624, + "device": 5, + "inactive_split_bytes_current": 5495342080, + "inactive_split_bytes_peak": 7158320128, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107017666560, + "reserved_bytes_peak": 107017666560, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:11:50.804444Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6011241472, + "inactive_split_bytes_current_delta": 515899392, + "inactive_split_bytes_peak": 7158320128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107487428608, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107487428608, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 546, + "segment_count_current_delta": 1, + "segment_count_peak": 546, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T121151890096Z.pickle", + "timestamp_utc": "2026-08-01T12:11:51.994832Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T121151890096Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T121151890096Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..38970e9c78cfe7c720062347da825ff6a123aeee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T121151890096Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a7428ada008d9910c8173f2499307a100059b01614ff2785fc60f19221440c8 +size 5533405 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T121154485640Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T121154485640Z.json new file mode 100644 index 0000000000000000000000000000000000000000..badbc50f40de3c0854dd24de2ba73c993d14d0bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T121154485640Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123991678976, + "active_bytes_peak_delta": 18163492352, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_peak": 8890998784, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137019523072, + "reserved_bytes_peak": 137019523072, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:54.485098Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105837920256, + "allocated_bytes_peak_delta": 9733632, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105828186624, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105828186624, + "device": 5, + "inactive_split_bytes_current": 6011241472, + "inactive_split_bytes_peak": 7158320128, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107487428608, + "reserved_bytes_peak": 107487428608, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T12:11:52.009610Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_current_delta": -535040000, + "inactive_split_bytes_peak": 8890998784, + "inactive_split_bytes_peak_delta": 1732678656, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137019523072, + "reserved_bytes_current_delta": 29532094464, + "reserved_bytes_peak": 137019523072, + "reserved_bytes_peak_delta": 29532094464, + "segment_count_current": 749, + "segment_count_current_delta": 203, + "segment_count_peak": 749, + "segment_count_peak_delta": 203, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T121154485640Z.pickle", + "timestamp_utc": "2026-08-01T12:11:54.607346Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T121154485640Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T121154485640Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f7ce6212c9d9877034fa918df8375937f419a92e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T121154485640Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55ca793c8dc3fa34c5e3393ce2b9b50b408062209629a57cf255d9bd03cc7953 +size 5832079 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T121155334886Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T121155334886Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6b4f6740513fc5fef8a3feab96290fb43c1cf6cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T121155334886Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991678976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_peak": 8890998784, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137019523072, + "reserved_bytes_peak": 137019523072, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:55.334411Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837920256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_peak": 8890998784, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137019523072, + "reserved_bytes_peak": 137019523072, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:54.624063Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8890998784, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137019523072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137019523072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T121155334886Z.pickle", + "timestamp_utc": "2026-08-01T12:11:55.341660Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T121155334886Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T121155334886Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c6908bbd781e523c25ae96882c15e086642a9adf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T121155334886Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49ae46c73b5c77166d96cacb6344d0ae308f82eb928e7d5e9abf55642be85f22 +size 222058 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T121156051321Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T121156051321Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e027ac833488d91e2e18027b373038ff29de1be9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T121156051321Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991678976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_peak": 8890998784, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137019523072, + "reserved_bytes_peak": 137019523072, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:56.050908Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837920256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_peak": 8890998784, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137019523072, + "reserved_bytes_peak": 137019523072, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:55.342642Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8890998784, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137019523072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137019523072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T121156051321Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.057512Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T121156051321Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T121156051321Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1f715ab3af0439432926c2a90bad5664e9a8104 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T121156051321Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4650925a28e688e667b5ad0d931922c3bc21fe5f447473f10214d0084abce2a2 +size 222027 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T121156766054Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T121156766054Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f687716cc406f1dc87bbfd17fbfc39f30771ebf5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T121156766054Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991678976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_peak": 8890998784, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137019523072, + "reserved_bytes_peak": 137019523072, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:56.765648Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837920256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_peak": 8890998784, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137019523072, + "reserved_bytes_peak": 137019523072, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:56.058444Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8890998784, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137019523072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137019523072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T121156766054Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.772706Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T121156766054Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T121156766054Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c1897aeccbb9f378e4922cda6f5b429fc0c3bc9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T121156766054Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a16e5294578f232bc07834dd711c543eb0b00a30c21a67765d3a0194e557bb8d +size 222027 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T121157480891Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T121157480891Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d2f0026ceff32d68429f24e59498c5c08bf6fd8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T121157480891Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991678976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_peak": 8890998784, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137019523072, + "reserved_bytes_peak": 137019523072, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:57.480484Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837920256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_peak": 8890998784, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137019523072, + "reserved_bytes_peak": 137019523072, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:56.773637Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8890998784, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137019523072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137019523072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T121157480891Z.pickle", + "timestamp_utc": "2026-08-01T12:11:57.487099Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T121157480891Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T121157480891Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b8771c723cace99e9f08758ec4dbbdbcf994fd6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T121157480891Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72a48722cde09132a2785d17b2a9018c0694d6132c55e9a8cbe70924c989948f +size 222027 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T121158197042Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T121158197042Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a2bde10eabccca5bd685ab03f5ac70e20f296f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T121158197042Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991678976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_peak": 8890998784, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137019523072, + "reserved_bytes_peak": 137019523072, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:58.196648Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837920256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_peak": 8890998784, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137019523072, + "reserved_bytes_peak": 137019523072, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:57.488026Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8890998784, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137019523072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137019523072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T121158197042Z.pickle", + "timestamp_utc": "2026-08-01T12:11:58.202936Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T121158197042Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T121158197042Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f6c9c5bd7d78f28028634470204ce0b0f3ce8f9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T121158197042Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92eb373e7887dd5811cf3dfb4f78493faedf217d27ed4662ed21f474418c8106 +size 222027 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T121247606928Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T121247606928Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a53ef94c32012a13fd3203b6838ea29bc0fa6fc3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T121247606928Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854491136, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 123991678976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854491136, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66854491136, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 3087625216, + "inactive_split_bytes_peak": 8890998784, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 137042591744, + "reserved_bytes_peak": 137042591744, + "segment_count_current": 751, + "segment_count_peak": 751, + "timestamp_utc": "2026-08-01T12:12:47.606431Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854491136, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105837920256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991678976, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837920256, + "device": 5, + "inactive_split_bytes_current": 5476201472, + "inactive_split_bytes_peak": 8890998784, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 137019523072, + "reserved_bytes_peak": 137019523072, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:58.203845Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3087625216, + "inactive_split_bytes_current_delta": -2388576256, + "inactive_split_bytes_peak": 8890998784, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 137042591744, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 137042591744, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 751, + "segment_count_current_delta": 2, + "segment_count_peak": 751, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T121247606928Z.pickle", + "timestamp_utc": "2026-08-01T12:12:47.691079Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T121247606928Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T121247606928Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..297c717fa170da25a9d0d13aadf5ed43a90229a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T121247606928Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fe772abefa5d6d20ae00fdcc3c72d7810f25ea7cf26f2bb8b42a3a9b04c3648 +size 4014554 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T121149508276Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T121149508276Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e27dc143a569d2a61444ed8361f75724e47b26ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T121149508276Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 31639626752, + "active_bytes_peak": 82194388480, + "active_bytes_peak_delta": 46728052224, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 82194388480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 82194388480, + "device": 6, + "inactive_split_bytes_current": 4912333824, + "inactive_split_bytes_peak": 7271640064, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:11:49.507576Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 31639626752, + "allocated_bytes_peak": 82194388480, + "allocated_bytes_peak_delta": 46728052224, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:10:41.423676Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4912333824, + "inactive_split_bytes_current_delta": 4779514880, + "inactive_split_bytes_peak": 7271640064, + "inactive_split_bytes_peak_delta": 7111254016, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 488, + "segment_count_current_delta": 417, + "segment_count_peak": 488, + "segment_count_peak_delta": 417, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T121149508276Z.pickle", + "timestamp_utc": "2026-08-01T12:11:49.622546Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T121149508276Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T121149508276Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5209c781df94a660df29f6dddd0c5003a2bf2bd9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T121149508276Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c12f084aa232be0b318b0882b8d13b4d0f012c21744545b788d700e89cb4fcc6 +size 5373786 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T121150684004Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T121150684004Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c1130fbe7692f36ba246f1d6bd7fc7c754f4cb72 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T121150684004Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105826068480, + "active_bytes_peak_delta": 23631680000, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 6, + "inactive_split_bytes_current": 4734075904, + "inactive_split_bytes_peak": 7271640064, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:50.683587Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105826068480, + "allocated_bytes_peak_delta": 23631680000, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 82194388480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 82194388480, + "device": 6, + "inactive_split_bytes_current": 4912333824, + "inactive_split_bytes_peak": 7271640064, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:11:49.642286Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4734075904, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 7271640064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107047026688, + "reserved_bytes_current_delta": 9558818816, + "reserved_bytes_peak": 107047026688, + "reserved_bytes_peak_delta": 9558818816, + "segment_count_current": 543, + "segment_count_current_delta": 55, + "segment_count_peak": 543, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T121150684004Z.pickle", + "timestamp_utc": "2026-08-01T12:11:50.788332Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T121150684004Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T121150684004Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e71f58c21a28f9a7f08ee30c126fb67bb7802660 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T121150684004Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb88712189669ff510b658cae46ad0f7ac73f34e440680de78ab564e83821746 +size 5532127 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T121151890037Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T121151890037Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f09570909801a39b7ac9aaf382ba6798c2b54d24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T121151890037Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871081984, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105826068480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 6, + "inactive_split_bytes_current": 5856052224, + "inactive_split_bytes_peak": 7271640064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:51.889631Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871081984, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105826068480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 6, + "inactive_split_bytes_current": 4734075904, + "inactive_split_bytes_peak": 7271640064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:50.803474Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5856052224, + "inactive_split_bytes_current_delta": 1121976320, + "inactive_split_bytes_peak": 7271640064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107047026688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107047026688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 543, + "segment_count_current_delta": 0, + "segment_count_peak": 543, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T121151890037Z.pickle", + "timestamp_utc": "2026-08-01T12:11:51.994462Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T121151890037Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T121151890037Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..654aaa18f0bf59f1c23bb93f9adcc9e08cf09658 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T121151890037Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1b3fa43f802ce56024ec931086db4723175be954e1b8742cc603441a8e3a177 +size 5551880 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T121154485949Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T121154485949Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f61d5777053c9e353dc5b675ac9aabf2e19ca513 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T121154485949Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 18165348352, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 9252860928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 136864333824, + "reserved_bytes_peak": 136864333824, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:54.485298Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 11589632, + "before": { + "active_bytes_current": 66871081984, + "active_bytes_peak": 105826068480, + "allocated_bytes_current": 66871081984, + "allocated_bytes_peak": 105826068480, + "device": 6, + "inactive_split_bytes_current": 5856052224, + "inactive_split_bytes_peak": 7271640064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107047026688, + "reserved_bytes_peak": 107047026688, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:11:52.009005Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_current_delta": -115609600, + "inactive_split_bytes_peak": 9252860928, + "inactive_split_bytes_peak_delta": 1981220864, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 136864333824, + "reserved_bytes_current_delta": 29817307136, + "reserved_bytes_peak": 136864333824, + "reserved_bytes_peak_delta": 29817307136, + "segment_count_current": 749, + "segment_count_current_delta": 206, + "segment_count_peak": 749, + "segment_count_peak_delta": 206, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T121154485949Z.pickle", + "timestamp_utc": "2026-08-01T12:11:54.604972Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T121154485949Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T121154485949Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d8e9d88f7e042f68c8b101346e516925575d9ddc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T121154485949Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06a817badf72d18b99f62b37690feddde3ae0c6522f663416350d31c9d3035a1 +size 5831818 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T121155334892Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T121155334892Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fbd4b1cf318c8f71191cf278d5e91e698ad6d82c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T121155334892Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 9252860928, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 136864333824, + "reserved_bytes_peak": 136864333824, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:55.334392Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 9252860928, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 136864333824, + "reserved_bytes_peak": 136864333824, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:54.621739Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9252860928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 136864333824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136864333824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T121155334892Z.pickle", + "timestamp_utc": "2026-08-01T12:11:55.341654Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T121155334892Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T121155334892Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f2eb176f52d973098eb0c876de1f478fa2cb200f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T121155334892Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5b931afc280804b73f721f98c324dcc7e48728d1992aaa4518cd8036e626c59 +size 222035 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T121156051313Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T121156051313Z.json new file mode 100644 index 0000000000000000000000000000000000000000..93ea8eab408567922b725c21a837c7e584303b30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T121156051313Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 9252860928, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 136864333824, + "reserved_bytes_peak": 136864333824, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:56.050892Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 9252860928, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 136864333824, + "reserved_bytes_peak": 136864333824, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:55.342610Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9252860928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 136864333824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136864333824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T121156051313Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.057788Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T121156051313Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T121156051313Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ab83a0215277e11085ab4246d8124fcbfbbaec96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T121156051313Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a3da307737a9e500527ee19d4476871ce183835a55250ef0ea5718c3115172a +size 222035 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T121156765912Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T121156765912Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b62907659dc8eda059e5509da0591134d78c9f87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T121156765912Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 9252860928, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 136864333824, + "reserved_bytes_peak": 136864333824, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:56.765531Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 9252860928, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 136864333824, + "reserved_bytes_peak": 136864333824, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:56.058748Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9252860928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 136864333824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136864333824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T121156765912Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.771675Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T121156765912Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T121156765912Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ab47456208148990aeef6f37bde289ab7d39ebec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T121156765912Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f67e5e83ad64a51dc316c2b22f7e33bfd80fb93f8c1bf584fa03d8d190c72f7b +size 222035 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T121157480859Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T121157480859Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0e18f4eef854a6848a385c356e52c610bbdfd90f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T121157480859Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 9252860928, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 136864333824, + "reserved_bytes_peak": 136864333824, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:57.480456Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 9252860928, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 136864333824, + "reserved_bytes_peak": 136864333824, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:56.772582Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9252860928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 136864333824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136864333824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T121157480859Z.pickle", + "timestamp_utc": "2026-08-01T12:11:57.486979Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T121157480859Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T121157480859Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e27bf975ed0807778b9e1bd86f2e681a4ce49ae5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T121157480859Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db3cd725aeeb688361279c886d0120352aed531578b70b2667ec63c0e571516d +size 222035 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T121158197015Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T121158197015Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27688bdb5879d3c2b411f36e165047a8e057a59f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T121158197015Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 9252860928, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 136864333824, + "reserved_bytes_peak": 136864333824, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:58.196629Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 9252860928, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 136864333824, + "reserved_bytes_peak": 136864333824, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:57.487927Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9252860928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 136864333824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 136864333824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T121158197015Z.pickle", + "timestamp_utc": "2026-08-01T12:11:58.203158Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T121158197015Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T121158197015Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3463a2e904b0760df18407bbc4a93975393278e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T121158197015Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:486e514c626430c18fb3b2e0ddfd8c72b480709b979bf758a58e89f411162530 +size 222035 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T121247606912Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T121247606912Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f01127e48985036948801132115435db586e2e92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T121247606912Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854491136, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 123991416832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854491136, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66854491136, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 2968087552, + "inactive_split_bytes_peak": 9252860928, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 137300541440, + "reserved_bytes_peak": 137300541440, + "segment_count_current": 752, + "segment_count_peak": 752, + "timestamp_utc": "2026-08-01T12:12:47.606466Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854491136, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105837658112, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125440, + "active_bytes_peak": 123991416832, + "allocated_bytes_current": 66888125440, + "allocated_bytes_peak": 105837658112, + "device": 6, + "inactive_split_bytes_current": 5740442624, + "inactive_split_bytes_peak": 9252860928, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 136864333824, + "reserved_bytes_peak": 136864333824, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:11:58.204055Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2968087552, + "inactive_split_bytes_current_delta": -2772355072, + "inactive_split_bytes_peak": 9252860928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 137300541440, + "reserved_bytes_current_delta": 436207616, + "reserved_bytes_peak": 137300541440, + "reserved_bytes_peak_delta": 436207616, + "segment_count_current": 752, + "segment_count_current_delta": 3, + "segment_count_peak": 752, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T121247606912Z.pickle", + "timestamp_utc": "2026-08-01T12:12:47.699655Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T121247606912Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T121247606912Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..30fecce3e852c1cacdb8768c71ad7539fdb6fc30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T121247606912Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72d1573719d56946bccc5c2a8ad411815b12a95aac56a475cb19a4f9c6f69a76 +size 4014621 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T121149508279Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T121149508279Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e93904b65cb6e8479dbc6a1ea187d67ca9afb1a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T121149508279Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871082496, + "active_bytes_current_delta": 31639627264, + "active_bytes_peak": 82194388480, + "active_bytes_peak_delta": 46728052224, + "after": { + "active_bytes_current": 66871082496, + "active_bytes_peak": 82194388480, + "allocated_bytes_current": 66871082496, + "allocated_bytes_peak": 82194388480, + "device": 7, + "inactive_split_bytes_current": 4916527616, + "inactive_split_bytes_peak": 7158402048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:11:49.507583Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871082496, + "allocated_bytes_current_delta": 31639627264, + "allocated_bytes_peak": 82194388480, + "allocated_bytes_peak_delta": 46728052224, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:10:41.423736Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4916527616, + "inactive_split_bytes_current_delta": 4783708672, + "inactive_split_bytes_peak": 7158402048, + "inactive_split_bytes_peak_delta": 6998016000, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 488, + "segment_count_current_delta": 417, + "segment_count_peak": 488, + "segment_count_peak_delta": 417, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T121149508279Z.pickle", + "timestamp_utc": "2026-08-01T12:11:49.619197Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T121149508279Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T121149508279Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c852e31bc5b299938064681ca4acc5e918ba0310 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T121149508279Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:455d834c12e8978350275cb43ff942de71ec59fda74f242a8519dc16a8d01a06 +size 5375184 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T121150684007Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T121150684007Z.json new file mode 100644 index 0000000000000000000000000000000000000000..499143b9a86c10a414dd4ebe276f6d6200cfd201 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T121150684007Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871082496, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105828187136, + "active_bytes_peak_delta": 23633798656, + "after": { + "active_bytes_current": 66871082496, + "active_bytes_peak": 105828187136, + "allocated_bytes_current": 66871082496, + "allocated_bytes_peak": 105828187136, + "device": 7, + "inactive_split_bytes_current": 5503730176, + "inactive_split_bytes_peak": 7158402048, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107017666560, + "reserved_bytes_peak": 107017666560, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:11:50.683593Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871082496, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105828187136, + "allocated_bytes_peak_delta": 23633798656, + "before": { + "active_bytes_current": 66871082496, + "active_bytes_peak": 82194388480, + "allocated_bytes_current": 66871082496, + "allocated_bytes_peak": 82194388480, + "device": 7, + "inactive_split_bytes_current": 4916527616, + "inactive_split_bytes_peak": 7158402048, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:11:49.638781Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5503730176, + "inactive_split_bytes_current_delta": 587202560, + "inactive_split_bytes_peak": 7158402048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107017666560, + "reserved_bytes_current_delta": 9529458688, + "reserved_bytes_peak": 107017666560, + "reserved_bytes_peak_delta": 9529458688, + "segment_count_current": 545, + "segment_count_current_delta": 57, + "segment_count_peak": 545, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T121150684007Z.pickle", + "timestamp_utc": "2026-08-01T12:11:50.788233Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T121150684007Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T121150684007Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..820e3652ebef67505dba5a799a9ca984af3df6d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T121150684007Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3553ed213d8c0ce44e81b27e50670411b9432a80db1e3dad688fe17244d810ef +size 5527687 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T121151890077Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T121151890077Z.json new file mode 100644 index 0000000000000000000000000000000000000000..468d5ad4d23b2b7c5d9555dd120413b59ad6d12f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T121151890077Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871082496, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105828187136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871082496, + "active_bytes_peak": 105828187136, + "allocated_bytes_current": 66871082496, + "allocated_bytes_peak": 105828187136, + "device": 7, + "inactive_split_bytes_current": 6007046656, + "inactive_split_bytes_peak": 7158402048, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107487428608, + "reserved_bytes_peak": 107487428608, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T12:11:51.889653Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871082496, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105828187136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871082496, + "active_bytes_peak": 105828187136, + "allocated_bytes_current": 66871082496, + "allocated_bytes_peak": 105828187136, + "device": 7, + "inactive_split_bytes_current": 5503730176, + "inactive_split_bytes_peak": 7158402048, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107017666560, + "reserved_bytes_peak": 107017666560, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:11:50.803227Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6007046656, + "inactive_split_bytes_current_delta": 503316480, + "inactive_split_bytes_peak": 7158402048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107487428608, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107487428608, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 546, + "segment_count_current_delta": 1, + "segment_count_peak": 546, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T121151890077Z.pickle", + "timestamp_utc": "2026-08-01T12:11:51.994778Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T121151890077Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T121151890077Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe53100c1746fe9a16e959f2f828ab4453dcf6c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T121151890077Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98776361dbbf33889e896d7adecfca09730d21af192a9846f6d4d5e13fae77f3 +size 5535404 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T121154485694Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T121154485694Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e2ce458509e02cb49cd18f627ac75db138d862f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T121154485694Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125952, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123991679488, + "active_bytes_peak_delta": 18163492352, + "after": { + "active_bytes_current": 66888125952, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66888125952, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_peak": 8901484032, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137015328768, + "reserved_bytes_peak": 137015328768, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:54.485120Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125952, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105837920768, + "allocated_bytes_peak_delta": 9733632, + "before": { + "active_bytes_current": 66871082496, + "active_bytes_peak": 105828187136, + "allocated_bytes_current": 66871082496, + "allocated_bytes_peak": 105828187136, + "device": 7, + "inactive_split_bytes_current": 6007046656, + "inactive_split_bytes_peak": 7158402048, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107487428608, + "reserved_bytes_peak": 107487428608, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T12:11:52.009295Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_current_delta": -524554240, + "inactive_split_bytes_peak": 8901484032, + "inactive_split_bytes_peak_delta": 1743081984, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137015328768, + "reserved_bytes_current_delta": 29527900160, + "reserved_bytes_peak": 137015328768, + "reserved_bytes_peak_delta": 29527900160, + "segment_count_current": 747, + "segment_count_current_delta": 201, + "segment_count_peak": 747, + "segment_count_peak_delta": 201, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T121154485694Z.pickle", + "timestamp_utc": "2026-08-01T12:11:54.603616Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T121154485694Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T121154485694Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..69cb2b7a9af9a7da6ca837fdd8f9c2ef355c3fc3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T121154485694Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f141d7bf0a0d0f6f1fb26259a527771373e6702001726b92657565d7bd0dfcf3 +size 5833814 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T121155334970Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T121155334970Z.json new file mode 100644 index 0000000000000000000000000000000000000000..09845e4f7025e4608e8e85d04c0a54d6e8344eab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T121155334970Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125952, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991679488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125952, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66888125952, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_peak": 8901484032, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137015328768, + "reserved_bytes_peak": 137015328768, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:55.334491Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125952, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837920768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125952, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66888125952, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_peak": 8901484032, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137015328768, + "reserved_bytes_peak": 137015328768, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:54.619755Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8901484032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137015328768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137015328768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T121155334970Z.pickle", + "timestamp_utc": "2026-08-01T12:11:55.341817Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T121155334970Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T121155334970Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..50cb85692a5283f476b63c318067c5375a2af98f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T121155334970Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bf2afb71e4015d9225d31c5589c15cca19ee4aa2b624f07fbb6ffb04f977907 +size 223711 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T121156051368Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T121156051368Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca81937ea51bbdbcbc2498d5b349be5beb22e64e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T121156051368Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125952, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991679488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125952, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66888125952, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_peak": 8901484032, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137015328768, + "reserved_bytes_peak": 137015328768, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:56.050955Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125952, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837920768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125952, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66888125952, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_peak": 8901484032, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137015328768, + "reserved_bytes_peak": 137015328768, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:55.342769Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8901484032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137015328768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137015328768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T121156051368Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.057693Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T121156051368Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T121156051368Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..40b2c2478a9ca77d4e17f4ef3b435fc9d7583950 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T121156051368Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8dc5a6938ccb02a3c25c72de59c686535d854a40cb993bc21b16ede47e5b912 +size 223711 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T121156766011Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T121156766011Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42258bb8d596f2073a03c1370fc7fb7bce0222ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T121156766011Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125952, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991679488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125952, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66888125952, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_peak": 8901484032, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137015328768, + "reserved_bytes_peak": 137015328768, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:56.765616Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125952, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837920768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125952, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66888125952, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_peak": 8901484032, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137015328768, + "reserved_bytes_peak": 137015328768, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:56.058609Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8901484032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137015328768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137015328768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T121156766011Z.pickle", + "timestamp_utc": "2026-08-01T12:11:56.771835Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T121156766011Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T121156766011Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..30c3a4f9bbdab6a909cdef65c57e2218334a60c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T121156766011Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7ff53c34445b0a3bf9c87d22efdf51b06da3555f5dece24fffc9cea3394e169 +size 223680 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T121157480912Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T121157480912Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b9421cb4397e5c9440058f559b53cac9db258eae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T121157480912Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125952, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991679488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125952, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66888125952, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_peak": 8901484032, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137015328768, + "reserved_bytes_peak": 137015328768, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:57.480505Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125952, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837920768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125952, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66888125952, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_peak": 8901484032, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137015328768, + "reserved_bytes_peak": 137015328768, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:56.772745Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8901484032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137015328768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137015328768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T121157480912Z.pickle", + "timestamp_utc": "2026-08-01T12:11:57.486946Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T121157480912Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T121157480912Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b219076e983116b269a12d82f460f6c5fc7254a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T121157480912Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a0bea05ecef788861e8b7b1471e7f0c9be8dd0940a3e540d5c153ad6317a718 +size 223711 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T121158197069Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T121158197069Z.json new file mode 100644 index 0000000000000000000000000000000000000000..97cba040abc9fae1db7cbf9372a0a006e6c2ce9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T121158197069Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888125952, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123991679488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888125952, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66888125952, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_peak": 8901484032, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137015328768, + "reserved_bytes_peak": 137015328768, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:58.196685Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888125952, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105837920768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125952, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66888125952, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_peak": 8901484032, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137015328768, + "reserved_bytes_peak": 137015328768, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:57.487849Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8901484032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137015328768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137015328768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 747, + "segment_count_current_delta": 0, + "segment_count_peak": 747, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T121158197069Z.pickle", + "timestamp_utc": "2026-08-01T12:11:58.202960Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T121158197069Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T121158197069Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f9778d23cad777576e4a0bfb243dc3d9ff76250c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T121158197069Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bee3440695d7f776cacc6d1093f50a4184d9119b855b38e154df6e60778c5c4 +size 223711 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T121247606881Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T121247606881Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1040985bfeeb5b9e44dabc3bc45a03d3898cd97f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T121247606881Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854491648, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 123991679488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854491648, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66854491648, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 3093916160, + "inactive_split_bytes_peak": 8901484032, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 137038397440, + "reserved_bytes_peak": 137038397440, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:12:47.606432Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854491648, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105837920768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888125952, + "active_bytes_peak": 123991679488, + "allocated_bytes_current": 66888125952, + "allocated_bytes_peak": 105837920768, + "device": 7, + "inactive_split_bytes_current": 5482492416, + "inactive_split_bytes_peak": 8901484032, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 137015328768, + "reserved_bytes_peak": 137015328768, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:11:58.203868Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3093916160, + "inactive_split_bytes_current_delta": -2388576256, + "inactive_split_bytes_peak": 8901484032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 137038397440, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 137038397440, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 749, + "segment_count_current_delta": 2, + "segment_count_peak": 749, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T121247606881Z.pickle", + "timestamp_utc": "2026-08-01T12:12:47.695365Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T121247606881Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T121247606881Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ddbf95c50d279bf84deae3708acf2f31bfcb728 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T121247606881Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6119eefbe3ae1ad55cbb08ceddb67baf2367361890781e46a8a0134db56a13ca +size 4016242 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..82c04bfec2aec532f4c954e826e27fa83d7b1bb3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 2048 +capacity: 1.50 +expert_dispatch_overlap: off +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6009 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 236 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T121546235756Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T121546235756Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c4ea464a95520e16064c37abb32599edf7cb4f7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T121546235756Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457313280, + "active_bytes_current_delta": 37225858048, + "active_bytes_peak": 82162485760, + "active_bytes_peak_delta": 46696149504, + "after": { + "active_bytes_current": 72457313280, + "active_bytes_peak": 82162485760, + "allocated_bytes_current": 72457313280, + "allocated_bytes_peak": 82162485760, + "device": 0, + "inactive_split_bytes_current": 7467246592, + "inactive_split_bytes_peak": 8020220928, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 95621742592, + "reserved_bytes_peak": 95621742592, + "segment_count_current": 426, + "segment_count_peak": 426, + "timestamp_utc": "2026-08-01T12:15:46.235150Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457313280, + "allocated_bytes_current_delta": 37225858048, + "allocated_bytes_peak": 82162485760, + "allocated_bytes_peak_delta": 46696149504, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:14:37.407108Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7467246592, + "inactive_split_bytes_current_delta": 7332330496, + "inactive_split_bytes_peak": 8020220928, + "inactive_split_bytes_peak_delta": 7859589120, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 95621742592, + "reserved_bytes_current_delta": 56375640064, + "reserved_bytes_peak": 95621742592, + "reserved_bytes_peak_delta": 56375640064, + "segment_count_current": 426, + "segment_count_current_delta": 354, + "segment_count_peak": 426, + "segment_count_peak_delta": 354, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T121546235756Z.pickle", + "timestamp_utc": "2026-08-01T12:15:46.353141Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T121546235756Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T121546235756Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dbbdc0d556a035ef9b569b6acfdb3e359d269fa5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T121546235756Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3308f60dea11520813b968b344870188c1db41392c41ffec9dbeef91262052d +size 5933025 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T121547334027Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T121547334027Z.json new file mode 100644 index 0000000000000000000000000000000000000000..da1cfba7515aaf177f8ecf4e09130b9b4c373d8d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T121547334027Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457313280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105797212160, + "active_bytes_peak_delta": 23634726400, + "after": { + "active_bytes_current": 72457313280, + "active_bytes_peak": 105797212160, + "allocated_bytes_current": 72457313280, + "allocated_bytes_peak": 105797212160, + "device": 0, + "inactive_split_bytes_current": 6232024064, + "inactive_split_bytes_peak": 8020220928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107122524160, + "reserved_bytes_peak": 107122524160, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T12:15:47.333626Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457313280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105797212160, + "allocated_bytes_peak_delta": 23634726400, + "before": { + "active_bytes_current": 72457313280, + "active_bytes_peak": 82162485760, + "allocated_bytes_current": 72457313280, + "allocated_bytes_peak": 82162485760, + "device": 0, + "inactive_split_bytes_current": 7467246592, + "inactive_split_bytes_peak": 8020220928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 95621742592, + "reserved_bytes_peak": 95621742592, + "segment_count_current": 426, + "segment_count_peak": 426, + "timestamp_utc": "2026-08-01T12:15:46.372575Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6232024064, + "inactive_split_bytes_current_delta": -1235222528, + "inactive_split_bytes_peak": 8020220928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107122524160, + "reserved_bytes_current_delta": 11500781568, + "reserved_bytes_peak": 107122524160, + "reserved_bytes_peak_delta": 11500781568, + "segment_count_current": 483, + "segment_count_current_delta": 57, + "segment_count_peak": 483, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T121547334027Z.pickle", + "timestamp_utc": "2026-08-01T12:15:47.446791Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T121547334027Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T121547334027Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29c7a836f14e081a86c3c934f09e899337c3df53 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T121547334027Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7951e7e40e71748be001ce89d5bbcfc25372e9799fe449ee1b7ad7c8bbcc48eb +size 5979500 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T121548417466Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T121548417466Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b18ba891b8c6484a9f66585ac1ce150366c52b92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T121548417466Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457313280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105797212160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457313280, + "active_bytes_peak": 105797212160, + "allocated_bytes_current": 72457313280, + "allocated_bytes_peak": 105797212160, + "device": 0, + "inactive_split_bytes_current": 6827615232, + "inactive_split_bytes_peak": 8099874816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107122524160, + "reserved_bytes_peak": 107122524160, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T12:15:48.417063Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457313280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105797212160, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457313280, + "active_bytes_peak": 105797212160, + "allocated_bytes_current": 72457313280, + "allocated_bytes_peak": 105797212160, + "device": 0, + "inactive_split_bytes_current": 6232024064, + "inactive_split_bytes_peak": 8020220928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107122524160, + "reserved_bytes_peak": 107122524160, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T12:15:47.462667Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6827615232, + "inactive_split_bytes_current_delta": 595591168, + "inactive_split_bytes_peak": 8099874816, + "inactive_split_bytes_peak_delta": 79653888, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107122524160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107122524160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T121548417466Z.pickle", + "timestamp_utc": "2026-08-01T12:15:48.525867Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T121548417466Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T121548417466Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca3ec96a331cc5c326e665f0a4683614be5f07d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T121548417466Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ab0e45ee6772dd856040e47c45e00b9c9d280ca1682a7725a96b76c76144ee9 +size 6283689 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T121551081202Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T121551081202Z.json new file mode 100644 index 0000000000000000000000000000000000000000..913effe293f92a6575424e829a5edf864cc9f03a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T121551081202Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474356736, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123966135296, + "active_bytes_peak_delta": 18168923136, + "after": { + "active_bytes_current": 72474356736, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 72474356736, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_peak": 13746091008, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_peak": 138359603200, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:51.080647Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474356736, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105812376576, + "allocated_bytes_peak_delta": 15164416, + "before": { + "active_bytes_current": 72457313280, + "active_bytes_peak": 105797212160, + "allocated_bytes_current": 72457313280, + "allocated_bytes_peak": 105797212160, + "device": 0, + "inactive_split_bytes_current": 6827615232, + "inactive_split_bytes_peak": 8099874816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107122524160, + "reserved_bytes_peak": 107122524160, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T12:15:48.541267Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_current_delta": 5295042560, + "inactive_split_bytes_peak": 13746091008, + "inactive_split_bytes_peak_delta": 5646216192, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_current_delta": 31237079040, + "reserved_bytes_peak": 138359603200, + "reserved_bytes_peak_delta": 31237079040, + "segment_count_current": 633, + "segment_count_current_delta": 150, + "segment_count_peak": 633, + "segment_count_peak_delta": 150, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T121551081202Z.pickle", + "timestamp_utc": "2026-08-01T12:15:51.205280Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T121551081202Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T121551081202Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..12db436972b36f033eff6e418c9ca2a60ae889d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T121551081202Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bba36b59cecdd2a7204e928362957f89ca8fa0a05b9714f5975b93b8976373f1 +size 6316392 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T121552071051Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T121552071051Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4af6c62dd0d1cca667e69b85a67a832c7c8c4eea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T121552071051Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474356736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123966135296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474356736, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 72474356736, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_peak": 13746091008, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_peak": 138359603200, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:52.070175Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474356736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105812376576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474356736, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 72474356736, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_peak": 13746091008, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_peak": 138359603200, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:51.222837Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13746091008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138359603200, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 633, + "segment_count_current_delta": 0, + "segment_count_peak": 633, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T121552071051Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.075843Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T121552071051Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T121552071051Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff042ef92d8ce253e95b799b6c53540d8137acce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T121552071051Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e2e64b696b0d54ff4d43c011f5d27ca9fdf6a3764a659a4667f9c7721de5295 +size 168245 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T121552941980Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T121552941980Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e01fc193e26e60e2accf5e28a0413ba5a3e98c24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T121552941980Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474356736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123966135296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474356736, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 72474356736, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_peak": 13746091008, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_peak": 138359603200, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:52.941543Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474356736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105812376576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474356736, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 72474356736, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_peak": 13746091008, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_peak": 138359603200, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:52.076744Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13746091008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138359603200, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 633, + "segment_count_current_delta": 0, + "segment_count_peak": 633, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T121552941980Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.946567Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T121552941980Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T121552941980Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a88db87cdc6d8a1505c07eb11fd85214ec9c1469 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T121552941980Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81163f9579ddeb89f5459a34a603720a0b6104455d5e25eb28e7a57270d94d6c +size 168245 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T121553783563Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T121553783563Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd186ba06cd8601cc3aa6800bbfcf06880779bd1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T121553783563Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474356736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123966135296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474356736, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 72474356736, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_peak": 13746091008, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_peak": 138359603200, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:53.783171Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474356736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105812376576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474356736, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 72474356736, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_peak": 13746091008, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_peak": 138359603200, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:52.947490Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13746091008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138359603200, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 633, + "segment_count_current_delta": 0, + "segment_count_peak": 633, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T121553783563Z.pickle", + "timestamp_utc": "2026-08-01T12:15:53.787962Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T121553783563Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T121553783563Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a369cbacc276dd37563c7d18bb8cf82fa9d10a35 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T121553783563Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52c59fc018ef4dc547344428005d9de995e8161b6bcfb5b652a5d6315908753d +size 168245 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T121554625124Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T121554625124Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f74709f7e14cf7997f4ec80a8d87eab54f5ac9f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T121554625124Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474356736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123966135296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474356736, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 72474356736, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_peak": 13746091008, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_peak": 138359603200, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:54.624707Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474356736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105812376576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474356736, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 72474356736, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_peak": 13746091008, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_peak": 138359603200, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:53.788814Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13746091008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138359603200, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 633, + "segment_count_current_delta": 0, + "segment_count_peak": 633, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T121554625124Z.pickle", + "timestamp_utc": "2026-08-01T12:15:54.629786Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T121554625124Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T121554625124Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c08f74f105bafe47068903a80c59a9ee6f86dd8c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T121554625124Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be8387778d9a29703839005748161a49c9e048525b5a04d83c1ba1695ceb616f +size 168245 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T121555470272Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T121555470272Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e8f5ec709e5305bef1261110912e3f83caee44b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T121555470272Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474356736, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123966135296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474356736, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 72474356736, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_peak": 13746091008, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_peak": 138359603200, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:55.469830Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474356736, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105812376576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474356736, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 72474356736, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_peak": 13746091008, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_peak": 138359603200, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:54.630671Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13746091008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138359603200, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 633, + "segment_count_current_delta": 0, + "segment_count_peak": 633, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T121555470272Z.pickle", + "timestamp_utc": "2026-08-01T12:15:55.474940Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T121555470272Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T121555470272Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72137653cfd3421e917396e75c826d354120f5f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T121555470272Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac3712c3a1bc327317501cc5136b1e8697ddab34a4f726384246bbb022bce8a8 +size 168245 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T121647969487Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T121647969487Z.json new file mode 100644 index 0000000000000000000000000000000000000000..08484784b785b6ea75c026b44038c0aa38287be6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T121647969487Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61217076224, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 123966135296, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61217076224, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 61217076224, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 8488061952, + "inactive_split_bytes_peak": 13746091008, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 138382671872, + "reserved_bytes_peak": 138382671872, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:16:47.969011Z", + "world_size": 8 + }, + "allocated_bytes_current": 61217076224, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 105812376576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474356736, + "active_bytes_peak": 123966135296, + "allocated_bytes_current": 72474356736, + "allocated_bytes_peak": 105812376576, + "device": 0, + "inactive_split_bytes_current": 12122657792, + "inactive_split_bytes_peak": 13746091008, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 138359603200, + "reserved_bytes_peak": 138359603200, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:55.475799Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8488061952, + "inactive_split_bytes_current_delta": -3634595840, + "inactive_split_bytes_peak": 13746091008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 138382671872, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 138382671872, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 635, + "segment_count_current_delta": 2, + "segment_count_peak": 635, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T121647969487Z.pickle", + "timestamp_utc": "2026-08-01T12:16:48.055342Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T121647969487Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T121647969487Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c5844682c553aaf65a0c8856462d67d3f20b8be1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T121647969487Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1222db74b04aaf805c68b61adb2705821f0191d1ca389207e1659d1e31b82d2 +size 4085777 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T121546235667Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T121546235667Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a468086c32f0bff333bfbd9ced4b3c663f353a0c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T121546235667Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 82164424704, + "active_bytes_peak_delta": 46698088448, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 82164424704, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 82164424704, + "device": 1, + "inactive_split_bytes_current": 7001932800, + "inactive_split_bytes_peak": 7845398528, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 94218747904, + "reserved_bytes_peak": 94218747904, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.235077Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 82164424704, + "allocated_bytes_peak_delta": 46698088448, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:14:37.407480Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7001932800, + "inactive_split_bytes_current_delta": 6869113856, + "inactive_split_bytes_peak": 7845398528, + "inactive_split_bytes_peak_delta": 7685012480, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 94218747904, + "reserved_bytes_current_delta": 54974742528, + "reserved_bytes_peak": 94218747904, + "reserved_bytes_peak_delta": 54974742528, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T121546235667Z.pickle", + "timestamp_utc": "2026-08-01T12:15:46.348069Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T121546235667Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T121546235667Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c7ce5d4ef66f79b70ca17e1828a49d55d4978875 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T121546235667Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8130eeabc48586ff0b4f998594216926a76c1b9061f650ab0d1f20e672a8cc9 +size 5923419 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T121547333999Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T121547333999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0fa3cd7b82ec24f7caab87a629510f4f91aa2ebf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T121547333999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105804443648, + "active_bytes_peak_delta": 23640018944, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 1, + "inactive_split_bytes_current": 6775440384, + "inactive_split_bytes_peak": 7845398528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107095261184, + "reserved_bytes_peak": 107095261184, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T12:15:47.333594Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105804443648, + "allocated_bytes_peak_delta": 23640018944, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 82164424704, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 82164424704, + "device": 1, + "inactive_split_bytes_current": 7001932800, + "inactive_split_bytes_peak": 7845398528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 94218747904, + "reserved_bytes_peak": 94218747904, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.364874Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6775440384, + "inactive_split_bytes_current_delta": -226492416, + "inactive_split_bytes_peak": 7845398528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107095261184, + "reserved_bytes_current_delta": 12876513280, + "reserved_bytes_peak": 107095261184, + "reserved_bytes_peak_delta": 12876513280, + "segment_count_current": 480, + "segment_count_current_delta": 59, + "segment_count_peak": 480, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T121547333999Z.pickle", + "timestamp_utc": "2026-08-01T12:15:47.441348Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T121547333999Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T121547333999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c5d980533327a166c185b51a492993777eb6ccca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T121547333999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8add810ee4732579cf9e88db42a190fcbc5df8989ee9170e536aa136551ff61 +size 5881718 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T121548417477Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T121548417477Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03873af8b0d5d8b01c74802b5b598b16c7eba3ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T121548417477Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105804443648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 1, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_peak": 7845398528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107565023232, + "reserved_bytes_peak": 107565023232, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T12:15:48.417078Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105804443648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 1, + "inactive_split_bytes_current": 6775440384, + "inactive_split_bytes_peak": 7845398528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107095261184, + "reserved_bytes_peak": 107095261184, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T12:15:47.455015Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 7845398528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107565023232, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107565023232, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 481, + "segment_count_current_delta": 1, + "segment_count_peak": 481, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T121548417477Z.pickle", + "timestamp_utc": "2026-08-01T12:15:48.520143Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T121548417477Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T121548417477Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e0deadf2b63ff09502b8d71fb306bbf251bacfc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T121548417477Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b5b590f80efdce6eaee188bce795715766a539c2a4eab01f68dd291e96634cf +size 5885919 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T121551081173Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T121551081173Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e39e5f103da327e84ddd26300978698dc529e81f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T121551081173Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 18153704448, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_peak": 15986848768, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:51.080648Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 8595456, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 1, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_peak": 7845398528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107565023232, + "reserved_bytes_peak": 107565023232, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T12:15:48.533290Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_current_delta": 6008074240, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 8141450240, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 30740054016, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 30740054016, + "segment_count_current": 634, + "segment_count_current_delta": 153, + "segment_count_peak": 634, + "segment_count_peak_delta": 153, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T121551081173Z.pickle", + "timestamp_utc": "2026-08-01T12:15:51.201046Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T121551081173Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T121551081173Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..49e4ef57f5b754bd26044ceac7c9694460c14bd1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T121551081173Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcfe764d0e6cda0c2d8e410486232948b939eb7bd21671e1f9edc2d3c88fcafc +size 6170156 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T121552070606Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T121552070606Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d4b00eaa86c6a91a055d627d06c1ae196f834acc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T121552070606Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_peak": 15986848768, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.070152Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_peak": 15986848768, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:51.216647Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T121552070606Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.075253Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T121552070606Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T121552070606Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3cbf9542ec400a6b0945f68237f2b28d9a9036c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T121552070606Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:feb7e483cd42c42cd05fac3026f8be94013bf052b85bf7e34b5440a238122f49 +size 164283 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T121552941941Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T121552941941Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67a8c178bcbe3750ba27d4df996871e19b90c094 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T121552941941Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_peak": 15986848768, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.941535Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_peak": 15986848768, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.076120Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T121552941941Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.946383Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T121552941941Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T121552941941Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a6a85ae42c3e9f61ef903d7f8740e613d2760509 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T121552941941Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fbe879256fa87c795c6fa41e278e6b4365ae8359c517b43500797e2dedc8902 +size 164283 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T121553783695Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T121553783695Z.json new file mode 100644 index 0000000000000000000000000000000000000000..66df08cc3159dad1730105992bfe7dd512d46776 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T121553783695Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_peak": 15986848768, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:53.783233Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_peak": 15986848768, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.947249Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T121553783695Z.pickle", + "timestamp_utc": "2026-08-01T12:15:53.788131Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T121553783695Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T121553783695Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..86af081a7df5bbbd798007bc017882c7fad99647 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T121553783695Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7873e85ae99ff8dec9fc867185b36d049b180ead0f3b6f618ec4a57e003298cf +size 164283 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T121554625138Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T121554625138Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef879908e4f4f6ed2ddd12e7cc8e7f63331cef2b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T121554625138Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_peak": 15986848768, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:54.624724Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_peak": 15986848768, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:53.788964Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T121554625138Z.pickle", + "timestamp_utc": "2026-08-01T12:15:54.629452Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T121554625138Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T121554625138Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..98190a2b3aa31fbe8e153f085229597650cefaa1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T121554625138Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:178c5ac0c472dfca29c593e4f8952dbc785aaaec9c6b524be28e667eb687ff5d +size 164283 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T121555467667Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T121555467667Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea49a1875b28651f4532b1d7426c8e1dab66ae53 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T121555467667Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_peak": 15986848768, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:55.467275Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_peak": 15986848768, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:54.630315Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T121555467667Z.pickle", + "timestamp_utc": "2026-08-01T12:15:55.471865Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T121555467667Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T121555467667Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..92f4185d4d106f2d5ec5c6a3372bcf616be0608e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T121555467667Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3b821f3bc58cc8e5d777dbd621d6987abbb0db49cc71461de9cefbca2f40ac3 +size 164283 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T121647969495Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T121647969495Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab81e2dc75f948cb6d0b1ed10fef6e360e175465 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T121647969495Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 8851123200, + "inactive_split_bytes_peak": 15986848768, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 138328145920, + "reserved_bytes_peak": 138328145920, + "segment_count_current": 636, + "segment_count_peak": 636, + "timestamp_utc": "2026-08-01T12:16:47.969013Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 1, + "inactive_split_bytes_current": 12779320320, + "inactive_split_bytes_peak": 15986848768, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:55.472710Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8851123200, + "inactive_split_bytes_current_delta": -3928197120, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 138328145920, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 138328145920, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 636, + "segment_count_current_delta": 2, + "segment_count_peak": 636, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T121647969495Z.pickle", + "timestamp_utc": "2026-08-01T12:16:48.050994Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T121647969495Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T121647969495Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0b822056ffbf2a0c87ce0cedede5c6df4947804 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T121647969495Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4575647ab289f992499c8ed5a75821f2df9e0e66368051f8b6416a79c3fbf4b +size 3959528 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T121546235697Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T121546235697Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27e89b925fe915d508c25749cf26fda914ff9909 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T121546235697Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 82164424704, + "active_bytes_peak_delta": 46698088448, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 82164424704, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 82164424704, + "device": 2, + "inactive_split_bytes_current": 7001932800, + "inactive_split_bytes_peak": 7845398528, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 94218747904, + "reserved_bytes_peak": 94218747904, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.235084Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 82164424704, + "allocated_bytes_peak_delta": 46698088448, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:14:37.407472Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7001932800, + "inactive_split_bytes_current_delta": 6869113856, + "inactive_split_bytes_peak": 7845398528, + "inactive_split_bytes_peak_delta": 7685012480, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 94218747904, + "reserved_bytes_current_delta": 54974742528, + "reserved_bytes_peak": 94218747904, + "reserved_bytes_peak_delta": 54974742528, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T121546235697Z.pickle", + "timestamp_utc": "2026-08-01T12:15:46.352591Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T121546235697Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T121546235697Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d242b3e039053101f63255d37bbcbab138e754fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T121546235697Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae9df920c109c0987f14acde6f49e045120694805a77e8d466c99fabde67dcb4 +size 5923420 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T121547333982Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T121547333982Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e3b8f96b3bfe8dbf08a5cb41a9752777b34d3014 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T121547333982Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105804443648, + "active_bytes_peak_delta": 23640018944, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 2, + "inactive_split_bytes_current": 6775440384, + "inactive_split_bytes_peak": 7845398528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107095261184, + "reserved_bytes_peak": 107095261184, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T12:15:47.333572Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105804443648, + "allocated_bytes_peak_delta": 23640018944, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 82164424704, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 82164424704, + "device": 2, + "inactive_split_bytes_current": 7001932800, + "inactive_split_bytes_peak": 7845398528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 94218747904, + "reserved_bytes_peak": 94218747904, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.369462Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6775440384, + "inactive_split_bytes_current_delta": -226492416, + "inactive_split_bytes_peak": 7845398528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107095261184, + "reserved_bytes_current_delta": 12876513280, + "reserved_bytes_peak": 107095261184, + "reserved_bytes_peak_delta": 12876513280, + "segment_count_current": 480, + "segment_count_current_delta": 59, + "segment_count_peak": 480, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T121547333982Z.pickle", + "timestamp_utc": "2026-08-01T12:15:47.444196Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T121547333982Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T121547333982Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0bcbe6d96eb0b4140eec9e6d96d1e8c2ebd8fc00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T121547333982Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ff62a9ae55f96d62f23143e1e644f46685f79fb1d450cb2b49a575aa20b4570 +size 5881713 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T121548417506Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T121548417506Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb1584d7371122e4104f547196002c5d4b432a4a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T121548417506Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105804443648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 2, + "inactive_split_bytes_current": 6769148928, + "inactive_split_bytes_peak": 7845398528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107565023232, + "reserved_bytes_peak": 107565023232, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T12:15:48.417091Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105804443648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 2, + "inactive_split_bytes_current": 6775440384, + "inactive_split_bytes_peak": 7845398528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107095261184, + "reserved_bytes_peak": 107095261184, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T12:15:47.457898Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6769148928, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 7845398528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107565023232, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107565023232, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 481, + "segment_count_current_delta": 1, + "segment_count_peak": 481, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T121548417506Z.pickle", + "timestamp_utc": "2026-08-01T12:15:48.522575Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T121548417506Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T121548417506Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e9317d2d305e83b2a2863fc97a98048836cad1ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T121548417506Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fd2aa7b5b32439ff7529d8fd99e2bfc8bbeb90eb4da4bcf52ac8ee4f87890d3 +size 5885925 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T121551081215Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T121551081215Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac16b11a1b3274fbeb637fa1bf763c7f248945c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T121551081215Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 18153704448, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15986848768, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 138302980096, + "reserved_bytes_peak": 138302980096, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:51.080684Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 8595456, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 2, + "inactive_split_bytes_current": 6769148928, + "inactive_split_bytes_peak": 7845398528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107565023232, + "reserved_bytes_peak": 107565023232, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T12:15:48.536006Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 6008074240, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 8141450240, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 138302980096, + "reserved_bytes_current_delta": 30737956864, + "reserved_bytes_peak": 138302980096, + "reserved_bytes_peak_delta": 30737956864, + "segment_count_current": 633, + "segment_count_current_delta": 152, + "segment_count_peak": 633, + "segment_count_peak_delta": 152, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T121551081215Z.pickle", + "timestamp_utc": "2026-08-01T12:15:51.211058Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T121551081215Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T121551081215Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..373c3cb59e0e15ca682541cef61fff2cc5408027 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T121551081215Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bb9b424b0595a97a6cc7a7f0566728b660bd5a363668eafc38536ca1e86eb1d +size 6170081 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T121552070602Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T121552070602Z.json new file mode 100644 index 0000000000000000000000000000000000000000..df12909a0b9dcde13a6f6f05a4236a2e6daf38ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T121552070602Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15986848768, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 138302980096, + "reserved_bytes_peak": 138302980096, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:52.070154Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15986848768, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 138302980096, + "reserved_bytes_peak": 138302980096, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:51.229948Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 138302980096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138302980096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 633, + "segment_count_current_delta": 0, + "segment_count_peak": 633, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T121552070602Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.075326Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T121552070602Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T121552070602Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db9e9ef8628752d15aad64ed9e1b4f362919be05 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T121552070602Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7468dfd41ba25e6e84dfbf613c2533603a627800228e888f4b495d42a5ee304a +size 164156 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T121552941923Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T121552941923Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bc6a0c8e93172a80db79612664c6793930420b0b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T121552941923Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15986848768, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 138302980096, + "reserved_bytes_peak": 138302980096, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:52.941509Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15986848768, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 138302980096, + "reserved_bytes_peak": 138302980096, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:52.076227Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 138302980096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138302980096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 633, + "segment_count_current_delta": 0, + "segment_count_peak": 633, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T121552941923Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.946553Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T121552941923Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T121552941923Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..42d6bc5e3bc24a5568079c6940f49fc2502c3552 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T121552941923Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aa848076c67df962d7584f70ff5e5277c6d356ca76b936af806c5dd6d824996 +size 164156 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T121553783582Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T121553783582Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9f917a55b60bcf431266dbf3fc9e8fe51063ec81 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T121553783582Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15986848768, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 138302980096, + "reserved_bytes_peak": 138302980096, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:53.783187Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15986848768, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 138302980096, + "reserved_bytes_peak": 138302980096, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:52.947429Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 138302980096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138302980096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 633, + "segment_count_current_delta": 0, + "segment_count_peak": 633, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T121553783582Z.pickle", + "timestamp_utc": "2026-08-01T12:15:53.787883Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T121553783582Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T121553783582Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cfa43bbc8d748dbef606dfd22316ea12f4017582 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T121553783582Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d81d945c0190f74461a351b20ba44620d70a59798b17a949caebaededa72a6b8 +size 164156 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T121554625117Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T121554625117Z.json new file mode 100644 index 0000000000000000000000000000000000000000..995e0467f814bf54803f7ce5b07b2d8c7135ba08 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T121554625117Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15986848768, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 138302980096, + "reserved_bytes_peak": 138302980096, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:54.624701Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15986848768, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 138302980096, + "reserved_bytes_peak": 138302980096, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:53.788738Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 138302980096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138302980096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 633, + "segment_count_current_delta": 0, + "segment_count_peak": 633, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T121554625117Z.pickle", + "timestamp_utc": "2026-08-01T12:15:54.629569Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T121554625117Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T121554625117Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..08beab764c7363a42e4f3a3d9333a27595e3dcc8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T121554625117Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbdf9360f8fb4e7517d98a8a7519aced25cd63a2d9e9e632a870a42b612dbf63 +size 164156 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T121555467657Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T121555467657Z.json new file mode 100644 index 0000000000000000000000000000000000000000..adec1b3cf72fcdcee34ef5cc21be5cdb4aa7963a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T121555467657Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15986848768, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 138302980096, + "reserved_bytes_peak": 138302980096, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:55.467269Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15986848768, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 138302980096, + "reserved_bytes_peak": 138302980096, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:54.630484Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 138302980096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138302980096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 633, + "segment_count_current_delta": 0, + "segment_count_peak": 633, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T121555467657Z.pickle", + "timestamp_utc": "2026-08-01T12:15:55.471824Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T121555467657Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T121555467657Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ebefd504245a5f3e86ea539835391fe6847ad080 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T121555467657Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21e975382b7af709ff1655845491c35d10676438e25e68ffd84bec5d2f7f5b97 +size 164156 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T121647969503Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T121647969503Z.json new file mode 100644 index 0000000000000000000000000000000000000000..542de95e64d9c7df70712b743e0d2de100179c39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T121647969503Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 8849026048, + "inactive_split_bytes_peak": 15986848768, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 138326048768, + "reserved_bytes_peak": 138326048768, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:16:47.969014Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 2, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15986848768, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 138302980096, + "reserved_bytes_peak": 138302980096, + "segment_count_current": 633, + "segment_count_peak": 633, + "timestamp_utc": "2026-08-01T12:15:55.472668Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8849026048, + "inactive_split_bytes_current_delta": -3928197120, + "inactive_split_bytes_peak": 15986848768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 138326048768, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 138326048768, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 635, + "segment_count_current_delta": 2, + "segment_count_peak": 635, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T121647969503Z.pickle", + "timestamp_utc": "2026-08-01T12:16:48.049968Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T121647969503Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T121647969503Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c38a20d450ba4600eb80274ea65e26d64aa61a0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T121647969503Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a113f1b5500a3102e938da285600dff4deebe06b04c5259f460b132564232f45 +size 3959375 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T121546235720Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T121546235720Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61f4e0d1a735ca7858f2b742503c87be46ebc500 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T121546235720Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 82164424704, + "active_bytes_peak_delta": 46698088448, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 82164424704, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 82164424704, + "device": 3, + "inactive_split_bytes_current": 7001932800, + "inactive_split_bytes_peak": 7845480448, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 94218747904, + "reserved_bytes_peak": 94218747904, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.235105Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 82164424704, + "allocated_bytes_peak_delta": 46698088448, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:14:37.407566Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7001932800, + "inactive_split_bytes_current_delta": 6869113856, + "inactive_split_bytes_peak": 7845480448, + "inactive_split_bytes_peak_delta": 7685094400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 94218747904, + "reserved_bytes_current_delta": 54974742528, + "reserved_bytes_peak": 94218747904, + "reserved_bytes_peak_delta": 54974742528, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T121546235720Z.pickle", + "timestamp_utc": "2026-08-01T12:15:46.351050Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T121546235720Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T121546235720Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e498b8455d2b20e03025de3eb188af4ef4a2fe0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T121546235720Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1416e94f108e243f658844ed3a3c7f1c7be778e862d986ab8a04807b48e532b9 +size 5923422 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T121547333999Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T121547333999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1d81d426b89d3e28dfef5db0f2e0abb2db9c8c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T121547333999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105804443648, + "active_bytes_peak_delta": 23640018944, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 3, + "inactive_split_bytes_current": 6773343232, + "inactive_split_bytes_peak": 7845480448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107095261184, + "reserved_bytes_peak": 107095261184, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T12:15:47.333597Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105804443648, + "allocated_bytes_peak_delta": 23640018944, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 82164424704, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 82164424704, + "device": 3, + "inactive_split_bytes_current": 7001932800, + "inactive_split_bytes_peak": 7845480448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 94218747904, + "reserved_bytes_peak": 94218747904, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.368153Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6773343232, + "inactive_split_bytes_current_delta": -228589568, + "inactive_split_bytes_peak": 7845480448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107095261184, + "reserved_bytes_current_delta": 12876513280, + "reserved_bytes_peak": 107095261184, + "reserved_bytes_peak_delta": 12876513280, + "segment_count_current": 480, + "segment_count_current_delta": 59, + "segment_count_peak": 480, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T121547333999Z.pickle", + "timestamp_utc": "2026-08-01T12:15:47.441540Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T121547333999Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T121547333999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ab64f1a72435b1a60ff3a4996324de6e6b8fd24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T121547333999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88fa64bec143a25ad3fd343ab4b79b63e2b4e7fd3e86d72eeaecd675baa0cea2 +size 5881720 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T121548417477Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T121548417477Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aa7417e872a0b3c474e1186eba9634b653968125 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T121548417477Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105804443648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 3, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_peak": 7845480448, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107565023232, + "reserved_bytes_peak": 107565023232, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T12:15:48.417079Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105804443648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 3, + "inactive_split_bytes_current": 6773343232, + "inactive_split_bytes_peak": 7845480448, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107095261184, + "reserved_bytes_peak": 107095261184, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T12:15:47.455264Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7845480448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107565023232, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107565023232, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 481, + "segment_count_current_delta": 1, + "segment_count_peak": 481, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T121548417477Z.pickle", + "timestamp_utc": "2026-08-01T12:15:48.521304Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T121548417477Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T121548417477Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5f450b147e29488ebc9f63ea1c71876a9b43898 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T121548417477Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b561d56063170edad41acb3cd0b169e87f1c996a76818faaf6e6d827407a26ee +size 5885968 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T121551081230Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T121551081230Z.json new file mode 100644 index 0000000000000000000000000000000000000000..172aea4dc9c673ac79e9a060bb6166dacb99ab5a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T121551081230Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 18153704448, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:51.080670Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 8595456, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 3, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_peak": 7845480448, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107565023232, + "reserved_bytes_peak": 107565023232, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T12:15:48.534569Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 6005977088, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 8139271168, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 30740054016, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 30740054016, + "segment_count_current": 634, + "segment_count_current_delta": 153, + "segment_count_peak": 634, + "segment_count_peak_delta": 153, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T121551081230Z.pickle", + "timestamp_utc": "2026-08-01T12:15:51.201805Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T121551081230Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T121551081230Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e3668ba664ed7ccaebc599a799bf11b8a7e5158 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T121551081230Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2add258a30f805be04c8cb744813b9cd29e6adfcc34fa37279a8538ab7d02436 +size 6170094 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T121552070646Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T121552070646Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8150efa11faf2ff68f10fb6d2322c23b836ac7e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T121552070646Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.070172Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:51.217050Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T121552070646Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.075605Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T121552070646Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T121552070646Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5d1ad830f1e1e4a96d28d825a0602f3b50de016 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T121552070646Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7eb17466c1b4de48acf9d6396530c9ffc5ba166a0421f8feaf66ca5c3b65b86 +size 164221 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T121552941950Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T121552941950Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e7303eb8d93861c40aae0ee0fa966df3301cf620 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T121552941950Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.941542Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.076499Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T121552941950Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.946496Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T121552941950Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T121552941950Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..06ed811fc28fd23ceb8159b408cdb8b75d46ff59 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T121552941950Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eff6574245419b42fd166180f0a11a926b16d6f819d7f6e42d10ce1fbc0d7b93 +size 164221 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T121553783550Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T121553783550Z.json new file mode 100644 index 0000000000000000000000000000000000000000..abc8bede02346d8a1e8405b448d70d298eaeb195 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T121553783550Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:53.783168Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.947361Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T121553783550Z.pickle", + "timestamp_utc": "2026-08-01T12:15:53.787810Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T121553783550Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T121553783550Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7fd4d496317b48ec3fc8994fe7218bb899d25b7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T121553783550Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f11005b469359c202f44e1c42402f1bc728a37f6f1d9ffa52c134416499e9b64 +size 164221 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T121554625156Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T121554625156Z.json new file mode 100644 index 0000000000000000000000000000000000000000..df9371587a69aee6e45c8b629e58c333dcb4ed43 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T121554625156Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:54.624711Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:53.788663Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T121554625156Z.pickle", + "timestamp_utc": "2026-08-01T12:15:54.629663Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T121554625156Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T121554625156Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d7f0123114279870ed8ab79083ce8b9b4be8aea8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T121554625156Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc64031776218abaf3c83181f0639fc45cba9eee58bc9bf47d41251b9d6fba2f +size 164221 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T121555467660Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T121555467660Z.json new file mode 100644 index 0000000000000000000000000000000000000000..067e40ec7eb9e94a3ad697d8c8853a6148800633 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T121555467660Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:55.467267Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:54.630563Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T121555467660Z.pickle", + "timestamp_utc": "2026-08-01T12:15:55.471935Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T121555467660Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T121555467660Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..17fb137cc7bfa814c16a74f6ed8627930dfd411f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T121555467660Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f262af822a318e104027c66cbe7add7ffdc9e9db771d992df250706b1dbf8abc +size 164221 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T121647969522Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T121647969522Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a865454716558152147ca0e4392e43e3d9cb186 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T121647969522Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 8849026048, + "inactive_split_bytes_peak": 15984751616, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 138328145920, + "reserved_bytes_peak": 138328145920, + "segment_count_current": 636, + "segment_count_peak": 636, + "timestamp_utc": "2026-08-01T12:16:47.969047Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 3, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:55.472779Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8849026048, + "inactive_split_bytes_current_delta": -3928197120, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 138328145920, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 138328145920, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 636, + "segment_count_current_delta": 2, + "segment_count_peak": 636, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T121647969522Z.pickle", + "timestamp_utc": "2026-08-01T12:16:48.051410Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T121647969522Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T121647969522Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..36d417b4111f6d410eda6e1a25c1439953d0d1ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T121647969522Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b7e4b414740717c2a8824ffa71409c7d6e033f857bdc8b2188a80077e51d6dd +size 3959499 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T121546235677Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T121546235677Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bcd32630860f7f53db5934d16c3fdd38939185c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T121546235677Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72458778624, + "active_bytes_current_delta": 37226881024, + "active_bytes_peak": 82163507200, + "active_bytes_peak_delta": 46696728576, + "after": { + "active_bytes_current": 72458778624, + "active_bytes_peak": 82163507200, + "allocated_bytes_current": 72458778624, + "allocated_bytes_peak": 82163507200, + "device": 4, + "inactive_split_bytes_current": 6517868544, + "inactive_split_bytes_peak": 7308839936, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93761568768, + "reserved_bytes_peak": 93761568768, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.235097Z", + "world_size": 8 + }, + "allocated_bytes_current": 72458778624, + "allocated_bytes_current_delta": 37226881024, + "allocated_bytes_peak": 82163507200, + "allocated_bytes_peak_delta": 46696728576, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:14:37.407091Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6517868544, + "inactive_split_bytes_current_delta": 6385491968, + "inactive_split_bytes_peak": 7308839936, + "inactive_split_bytes_peak_delta": 7155430400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93761568768, + "reserved_bytes_current_delta": 54517563392, + "reserved_bytes_peak": 93761568768, + "reserved_bytes_peak_delta": 54517563392, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T121546235677Z.pickle", + "timestamp_utc": "2026-08-01T12:15:46.350882Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T121546235677Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T121546235677Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..99b464247b19bbcb44dc017014a294a4236bc1d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T121546235677Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fb23021bbe7d3bef123bb6637522875f7df9235c167523c0789be1d5989750f +size 5931677 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T121547334044Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T121547334044Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69ccbe35c919062e6e5aee017b0153bf584f4e75 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T121547334044Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72458778624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105798487040, + "active_bytes_peak_delta": 23634979840, + "after": { + "active_bytes_current": 72458778624, + "active_bytes_peak": 105798487040, + "allocated_bytes_current": 72458778624, + "allocated_bytes_peak": 105798487040, + "device": 4, + "inactive_split_bytes_current": 6637406208, + "inactive_split_bytes_peak": 7407755264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107099455488, + "reserved_bytes_peak": 107099455488, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T12:15:47.333638Z", + "world_size": 8 + }, + "allocated_bytes_current": 72458778624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105798487040, + "allocated_bytes_peak_delta": 23634979840, + "before": { + "active_bytes_current": 72458778624, + "active_bytes_peak": 82163507200, + "allocated_bytes_current": 72458778624, + "allocated_bytes_peak": 82163507200, + "device": 4, + "inactive_split_bytes_current": 6517868544, + "inactive_split_bytes_peak": 7308839936, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93761568768, + "reserved_bytes_peak": 93761568768, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.367710Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6637406208, + "inactive_split_bytes_current_delta": 119537664, + "inactive_split_bytes_peak": 7407755264, + "inactive_split_bytes_peak_delta": 98915328, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107099455488, + "reserved_bytes_current_delta": 13337886720, + "reserved_bytes_peak": 107099455488, + "reserved_bytes_peak_delta": 13337886720, + "segment_count_current": 484, + "segment_count_current_delta": 63, + "segment_count_peak": 484, + "segment_count_peak_delta": 63, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T121547334044Z.pickle", + "timestamp_utc": "2026-08-01T12:15:47.443304Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T121547334044Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T121547334044Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..216c9451da6ff1ba110954ca6ee0b2255b74dc7f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T121547334044Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d639f9f1892d3a4509a635c295fa62373f3f22311398a5b802040e063323aa7 +size 5908115 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T121548417473Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T121548417473Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb2d92cc4d0dbb205e0eb213234ceb100626f3e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T121548417473Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72458778624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105798487040, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72458778624, + "active_bytes_peak": 105798487040, + "allocated_bytes_current": 72458778624, + "allocated_bytes_peak": 105798487040, + "device": 4, + "inactive_split_bytes_current": 6163449856, + "inactive_split_bytes_peak": 7990205952, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107569217536, + "reserved_bytes_peak": 107569217536, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T12:15:48.417072Z", + "world_size": 8 + }, + "allocated_bytes_current": 72458778624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105798487040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72458778624, + "active_bytes_peak": 105798487040, + "allocated_bytes_current": 72458778624, + "allocated_bytes_peak": 105798487040, + "device": 4, + "inactive_split_bytes_current": 6637406208, + "inactive_split_bytes_peak": 7407755264, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107099455488, + "reserved_bytes_peak": 107099455488, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T12:15:47.457212Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6163449856, + "inactive_split_bytes_current_delta": -473956352, + "inactive_split_bytes_peak": 7990205952, + "inactive_split_bytes_peak_delta": 582450688, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107569217536, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107569217536, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 485, + "segment_count_current_delta": 1, + "segment_count_peak": 485, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T121548417473Z.pickle", + "timestamp_utc": "2026-08-01T12:15:48.521460Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T121548417473Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T121548417473Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..78993bf111557b33bf29c04202ad93b1f24a3e4a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T121548417473Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f23cdf3775c5a2ba554fffb7d658a5703553a4e8d6a28bb56181898d7866aef +size 5893355 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T121551081179Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T121551081179Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5fb4de0e4c569e404d5128d1aa789c32cf907331 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T121551081179Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475822080, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123963537408, + "active_bytes_peak_delta": 18165050368, + "after": { + "active_bytes_current": 72475822080, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 72475822080, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_peak": 13972436992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137699000320, + "reserved_bytes_peak": 137699000320, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:15:51.080641Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475822080, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105813054976, + "allocated_bytes_peak_delta": 14567936, + "before": { + "active_bytes_current": 72458778624, + "active_bytes_peak": 105798487040, + "allocated_bytes_current": 72458778624, + "allocated_bytes_peak": 105798487040, + "device": 4, + "inactive_split_bytes_current": 6163449856, + "inactive_split_bytes_peak": 7990205952, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107569217536, + "reserved_bytes_peak": 107569217536, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T12:15:48.534968Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_current_delta": 5953548288, + "inactive_split_bytes_peak": 13972436992, + "inactive_split_bytes_peak_delta": 5982231040, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137699000320, + "reserved_bytes_current_delta": 30129782784, + "reserved_bytes_peak": 137699000320, + "reserved_bytes_peak_delta": 30129782784, + "segment_count_current": 629, + "segment_count_current_delta": 144, + "segment_count_peak": 629, + "segment_count_peak_delta": 144, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T121551081179Z.pickle", + "timestamp_utc": "2026-08-01T12:15:51.202028Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T121551081179Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T121551081179Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c0c56abd9959d7574d79d7bb3b16be6b2efb13ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T121551081179Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29dcd445931de6cca33eaa5617824880ff16edbd064e74847128e56f15826e8c +size 6176960 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T121552070962Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T121552070962Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d87e3d03cfbd0a77da5a3f7f1661455be95fa545 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T121552070962Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475822080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123963537408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475822080, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 72475822080, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_peak": 13972436992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137699000320, + "reserved_bytes_peak": 137699000320, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:15:52.070177Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475822080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813054976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475822080, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 72475822080, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_peak": 13972436992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137699000320, + "reserved_bytes_peak": 137699000320, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:15:51.217523Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13972436992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137699000320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137699000320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 629, + "segment_count_current_delta": 0, + "segment_count_peak": 629, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T121552070962Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.075576Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T121552070962Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T121552070962Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cd68fe70aef6c6e86dbb02512a08a17a222cc762 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T121552070962Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e458a80cf24a886572e7c4b1fd7bd448684d8654453226c3b32cd60ccbd13fd +size 170652 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T121552941942Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T121552941942Z.json new file mode 100644 index 0000000000000000000000000000000000000000..00d3808b999be2996bc9b86d1358487e5a8a8a3f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T121552941942Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475822080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123963537408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475822080, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 72475822080, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_peak": 13972436992, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137699000320, + "reserved_bytes_peak": 137699000320, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:15:52.941523Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475822080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813054976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475822080, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 72475822080, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_peak": 13972436992, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137699000320, + "reserved_bytes_peak": 137699000320, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:15:52.076478Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13972436992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137699000320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137699000320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 629, + "segment_count_current_delta": 0, + "segment_count_peak": 629, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T121552941942Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.946623Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T121552941942Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T121552941942Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f969418f947dbf62e1abed43fdd73a25ae7a95ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T121552941942Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22ad444079ee5f38f23d899297717747afb6e160e33b7bfe4434f2846785ce13 +size 170652 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T121553783623Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T121553783623Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b99559edcd635731e72ecfdea60685db30cff7c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T121553783623Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475822080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123963537408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475822080, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 72475822080, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_peak": 13972436992, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137699000320, + "reserved_bytes_peak": 137699000320, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:15:53.783203Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475822080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813054976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475822080, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 72475822080, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_peak": 13972436992, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137699000320, + "reserved_bytes_peak": 137699000320, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:15:52.947559Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13972436992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137699000320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137699000320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 629, + "segment_count_current_delta": 0, + "segment_count_peak": 629, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T121553783623Z.pickle", + "timestamp_utc": "2026-08-01T12:15:53.788037Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T121553783623Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T121553783623Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..800491e090ffa539a94032e8f75612de74375a43 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T121553783623Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:651c669262a3981c049c23ec56328c4fa64d62917fe3d829a1bb10c9dac69c31 +size 170652 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T121554625090Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T121554625090Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d6047c1225981d1c43982c7d9c6c5dd1c59bce7e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T121554625090Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475822080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123963537408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475822080, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 72475822080, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_peak": 13972436992, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137699000320, + "reserved_bytes_peak": 137699000320, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:15:54.624690Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475822080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813054976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475822080, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 72475822080, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_peak": 13972436992, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137699000320, + "reserved_bytes_peak": 137699000320, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:15:53.788898Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13972436992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137699000320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137699000320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 629, + "segment_count_current_delta": 0, + "segment_count_peak": 629, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T121554625090Z.pickle", + "timestamp_utc": "2026-08-01T12:15:54.629566Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T121554625090Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T121554625090Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..61b690ddcbcaac292240868ce52091e3ef9bdacb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T121554625090Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f64baf167e64862afee889916e5d3148f4e47b99fd7a4af6d47545d0f86da4b8 +size 170652 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T121555467674Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T121555467674Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e2f704359ddb9dc97ce6c69e7239f8e20bce22c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T121555467674Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72475822080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123963537408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72475822080, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 72475822080, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_peak": 13972436992, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137699000320, + "reserved_bytes_peak": 137699000320, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:15:55.467277Z", + "world_size": 8 + }, + "allocated_bytes_current": 72475822080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813054976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475822080, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 72475822080, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_peak": 13972436992, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137699000320, + "reserved_bytes_peak": 137699000320, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:15:54.630458Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13972436992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137699000320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 137699000320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 629, + "segment_count_current_delta": 0, + "segment_count_peak": 629, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T121555467674Z.pickle", + "timestamp_utc": "2026-08-01T12:15:55.471984Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T121555467674Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T121555467674Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b4cffffd6c1315747c4c237762141ede10c4adeb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T121555467674Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f16861860e75727b836e055c341890f24e23fbf6616386cfb57b66d52259ecb +size 170652 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T121647969520Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T121647969520Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bad31499158d5cc8126225803f74346a5f8c2f62 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T121647969520Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61218541568, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 123963537408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61218541568, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 61218541568, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 8247521280, + "inactive_split_bytes_peak": 13972436992, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 137722068992, + "reserved_bytes_peak": 137722068992, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:16:47.969050Z", + "world_size": 8 + }, + "allocated_bytes_current": 61218541568, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 105813054976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72475822080, + "active_bytes_peak": 123963537408, + "allocated_bytes_current": 72475822080, + "allocated_bytes_peak": 105813054976, + "device": 4, + "inactive_split_bytes_current": 12116998144, + "inactive_split_bytes_peak": 13972436992, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 137699000320, + "reserved_bytes_peak": 137699000320, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:15:55.472835Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8247521280, + "inactive_split_bytes_current_delta": -3869476864, + "inactive_split_bytes_peak": 13972436992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 137722068992, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 137722068992, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 631, + "segment_count_current_delta": 2, + "segment_count_peak": 631, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T121647969520Z.pickle", + "timestamp_utc": "2026-08-01T12:16:48.051147Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T121647969520Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T121647969520Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a8c2be9f8034e7c9b213b1858f95cc6e3876a0f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T121647969520Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f66868d0d8cee140215b2351a4b908167b7005421687ca16e417bf884242424b +size 3965936 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T121546235858Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T121546235858Z.json new file mode 100644 index 0000000000000000000000000000000000000000..370cc0ff09fac44544237ea13209ca59c9c49217 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T121546235858Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 82164424704, + "active_bytes_peak_delta": 46698088448, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 82164424704, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 82164424704, + "device": 5, + "inactive_split_bytes_current": 7004029952, + "inactive_split_bytes_peak": 7845398528, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 94218747904, + "reserved_bytes_peak": 94218747904, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.235241Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 82164424704, + "allocated_bytes_peak_delta": 46698088448, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:14:37.407748Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7004029952, + "inactive_split_bytes_current_delta": 6871211008, + "inactive_split_bytes_peak": 7845398528, + "inactive_split_bytes_peak_delta": 7685012480, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 94218747904, + "reserved_bytes_current_delta": 54974742528, + "reserved_bytes_peak": 94218747904, + "reserved_bytes_peak_delta": 54974742528, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T121546235858Z.pickle", + "timestamp_utc": "2026-08-01T12:15:46.352671Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T121546235858Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T121546235858Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..232a7c649436b96b85b8d36a408068aa337976f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T121546235858Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff08c69dc16645a84541c2a153e4473b72d623431f8f46669f69d3bb1fc660fc +size 5923416 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T121547334003Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T121547334003Z.json new file mode 100644 index 0000000000000000000000000000000000000000..351f3757d68fac4a42d755eaa93dd6fcb02169a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T121547334003Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105804443648, + "active_bytes_peak_delta": 23640018944, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 5, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_peak": 7845398528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107095261184, + "reserved_bytes_peak": 107095261184, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T12:15:47.333595Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105804443648, + "allocated_bytes_peak_delta": 23640018944, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 82164424704, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 82164424704, + "device": 5, + "inactive_split_bytes_current": 7004029952, + "inactive_split_bytes_peak": 7845398528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 94218747904, + "reserved_bytes_peak": 94218747904, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.367947Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 7845398528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107095261184, + "reserved_bytes_current_delta": 12876513280, + "reserved_bytes_peak": 107095261184, + "reserved_bytes_peak_delta": 12876513280, + "segment_count_current": 480, + "segment_count_current_delta": 59, + "segment_count_peak": 480, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T121547334003Z.pickle", + "timestamp_utc": "2026-08-01T12:15:47.441855Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T121547334003Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T121547334003Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60ce33fb79b42542e856a04ec0fcaa9d1ab0dc87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T121547334003Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42dc0f1f2c29068b84a011e73c16624a23510eb1b39c45019b78e281fb69afab +size 5881701 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T121548417495Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T121548417495Z.json new file mode 100644 index 0000000000000000000000000000000000000000..16dfe5e1b13fb9f317ed8bddc98c8baad02ffbb5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T121548417495Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105804443648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 5, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_peak": 7845398528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107565023232, + "reserved_bytes_peak": 107565023232, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T12:15:48.417100Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105804443648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 5, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_peak": 7845398528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107095261184, + "reserved_bytes_peak": 107095261184, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T12:15:47.454167Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7845398528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107565023232, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107565023232, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 481, + "segment_count_current_delta": 1, + "segment_count_peak": 481, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T121548417495Z.pickle", + "timestamp_utc": "2026-08-01T12:15:48.516392Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T121548417495Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T121548417495Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0d16f30a14cf41f7ec54c4eabd5fc3b9b8dcaf79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T121548417495Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c71c69efe5158216510b922cb6f5524714dafbeb1a06b75be2b015da0e083e3f +size 5885961 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T121551081328Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T121551081328Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d606ddfcbe5d37769e08f9d5ab575ecb71732d21 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T121551081328Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 18153704448, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:51.080739Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 8595456, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 5, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_peak": 7845398528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107565023232, + "reserved_bytes_peak": 107565023232, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T12:15:48.527943Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 6005977088, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 8139353088, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 30740054016, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 30740054016, + "segment_count_current": 634, + "segment_count_current_delta": 153, + "segment_count_peak": 634, + "segment_count_peak_delta": 153, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T121551081328Z.pickle", + "timestamp_utc": "2026-08-01T12:15:51.200539Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T121551081328Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T121551081328Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..488f401bcc050458b3ee20a5adc03d4a01e7ef71 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T121551081328Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:befde1cc131d0c924b6b882a7d2b3568001fe5ae76dbd351b0e93b10239fc8c9 +size 6170093 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T121552070604Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T121552070604Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ebabcd009108b1c9b9085024f7f00552e4f9dff1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T121552070604Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.070152Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:51.214638Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T121552070604Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.077182Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T121552070604Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T121552070604Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5dded21a8309312dc73da73a2fad86df952aa429 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T121552070604Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:059d4d13c5886621b1b3fe7d68e9b2c21f7700ede87fa98d5676c3b8ad001e8f +size 164220 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T121552941953Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T121552941953Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ecb6da71a4700912d59abac1bea516c159098bbb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T121552941953Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.941551Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.078114Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T121552941953Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.946628Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T121552941953Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T121552941953Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a9e504390b4ce3ae017c5c0cfb60f66203b42e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T121552941953Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:673a54d0bc542e576bf1aaa0a73eae28379c4d565a0ce44ce2d10bc0f3e49791 +size 164220 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T121553783576Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T121553783576Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ff6f65642268201c7661543e979a9afcf4b1627 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T121553783576Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:53.783186Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.947531Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T121553783576Z.pickle", + "timestamp_utc": "2026-08-01T12:15:53.787829Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T121553783576Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T121553783576Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d08535dd56a93dbce01b40d2d4c3a32b9e3ee354 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T121553783576Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c9d8b36e9fcc04f304ad20dbb69160b81154e3ac7c7a2ed9c56792be6b18c0d +size 164220 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T121554625116Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T121554625116Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d87ef0e22915d8f14335e99cf574dc7a38832d14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T121554625116Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:54.624726Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:53.788686Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T121554625116Z.pickle", + "timestamp_utc": "2026-08-01T12:15:54.629401Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T121554625116Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T121554625116Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a7469a727ec9c2cc70eac991234e7c5d0b199e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T121554625116Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02fad5f7a1251251b25a1cb8b01f8f47d8da5ca44571259fa90123d567577b70 +size 164220 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T121555467836Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T121555467836Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab0bddeccef10264202a505eea110f2acc775384 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T121555467836Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:55.467435Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:54.630275Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T121555467836Z.pickle", + "timestamp_utc": "2026-08-01T12:15:55.473088Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T121555467836Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T121555467836Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8075ce33327b7dc3d02edb05b485f5d56d2b1aa8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T121555467836Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea9dd1a223b48b150ce3776cf68bf1881dbc41f61f70c69e48a3402ef11d21dd +size 164220 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T121647969483Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T121647969483Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b39d152605f7f302e69ac66b993da0f2d6a77855 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T121647969483Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 8849026048, + "inactive_split_bytes_peak": 15984751616, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 138328145920, + "reserved_bytes_peak": 138328145920, + "segment_count_current": 636, + "segment_count_peak": 636, + "timestamp_utc": "2026-08-01T12:16:47.969016Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 5, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:55.474019Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8849026048, + "inactive_split_bytes_current_delta": -3928197120, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 138328145920, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 138328145920, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 636, + "segment_count_current_delta": 2, + "segment_count_peak": 636, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T121647969483Z.pickle", + "timestamp_utc": "2026-08-01T12:16:48.053188Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T121647969483Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T121647969483Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e83c666e6c87f7c08793582ebdede5181d3344e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T121647969483Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:260a87dc3b5e2e62e27eba210bb1d36417c2f1dbc08071457f641ac577957e10 +size 3959503 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T121546235804Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T121546235804Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7bb684a030ef44cd3fb443b481695c93d0e50e86 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T121546235804Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 37225604096, + "active_bytes_peak": 82164424704, + "active_bytes_peak_delta": 46698088448, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 82164424704, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 82164424704, + "device": 6, + "inactive_split_bytes_current": 7001932800, + "inactive_split_bytes_peak": 7845398528, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 94218747904, + "reserved_bytes_peak": 94218747904, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.235181Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 37225604096, + "allocated_bytes_peak": 82164424704, + "allocated_bytes_peak_delta": 46698088448, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:14:37.407509Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7001932800, + "inactive_split_bytes_current_delta": 6869113856, + "inactive_split_bytes_peak": 7845398528, + "inactive_split_bytes_peak_delta": 7685012480, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 94218747904, + "reserved_bytes_current_delta": 54974742528, + "reserved_bytes_peak": 94218747904, + "reserved_bytes_peak_delta": 54974742528, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T121546235804Z.pickle", + "timestamp_utc": "2026-08-01T12:15:46.357556Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T121546235804Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T121546235804Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ddfaeddd0c631e8e2eca70c4c46f159cc1f2cf5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T121546235804Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64056dcad191039f923f5afbc5ff3f3bd6b2c29ccf0f99dd7d9ec694cbedcee0 +size 5923423 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T121547333980Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T121547333980Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c037e5735d6173debb2c284c021aa0b35b963fa2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T121547333980Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105804443648, + "active_bytes_peak_delta": 23640018944, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 6, + "inactive_split_bytes_current": 6773343232, + "inactive_split_bytes_peak": 7845398528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107095261184, + "reserved_bytes_peak": 107095261184, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T12:15:47.333568Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105804443648, + "allocated_bytes_peak_delta": 23640018944, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 82164424704, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 82164424704, + "device": 6, + "inactive_split_bytes_current": 7001932800, + "inactive_split_bytes_peak": 7845398528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 94218747904, + "reserved_bytes_peak": 94218747904, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.376814Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6773343232, + "inactive_split_bytes_current_delta": -228589568, + "inactive_split_bytes_peak": 7845398528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107095261184, + "reserved_bytes_current_delta": 12876513280, + "reserved_bytes_peak": 107095261184, + "reserved_bytes_peak_delta": 12876513280, + "segment_count_current": 480, + "segment_count_current_delta": 59, + "segment_count_peak": 480, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T121547333980Z.pickle", + "timestamp_utc": "2026-08-01T12:15:47.444214Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T121547333980Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T121547333980Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..febe3c806699412ceeac524997aabada63fbbad8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T121547333980Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8c4da0a22a8cbdedc363024fb7f7072f8f6e82e32fe88d86e23e2b3d68aea6f +size 5881710 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T121548417441Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T121548417441Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c1e57d998f4d7003a1a13542bc24caa419690587 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T121548417441Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105804443648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 6, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_peak": 7845398528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107565023232, + "reserved_bytes_peak": 107565023232, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T12:15:48.417033Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105804443648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 6, + "inactive_split_bytes_current": 6773343232, + "inactive_split_bytes_peak": 7845398528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107095261184, + "reserved_bytes_peak": 107095261184, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T12:15:47.460034Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7845398528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107565023232, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107565023232, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 481, + "segment_count_current_delta": 1, + "segment_count_peak": 481, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T121548417441Z.pickle", + "timestamp_utc": "2026-08-01T12:15:48.523853Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T121548417441Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T121548417441Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c601596bd11c039f5593842046e5f289dd2f003b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T121548417441Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5124d01f3c677b2511ee11a51109dbed1066949119aed52623892bff02fed9dd +size 5885951 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T121551081167Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T121551081167Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1396a696c49563e1ec2a7c1b08b11ea6cfec2485 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T121551081167Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 18153704448, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:51.080626Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 8595456, + "before": { + "active_bytes_current": 72457059328, + "active_bytes_peak": 105804443648, + "allocated_bytes_current": 72457059328, + "allocated_bytes_peak": 105804443648, + "device": 6, + "inactive_split_bytes_current": 6771246080, + "inactive_split_bytes_peak": 7845398528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107565023232, + "reserved_bytes_peak": 107565023232, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T12:15:48.539363Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 6005977088, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 8139353088, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 30740054016, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 30740054016, + "segment_count_current": 634, + "segment_count_current_delta": 153, + "segment_count_peak": 634, + "segment_count_peak_delta": 153, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T121551081167Z.pickle", + "timestamp_utc": "2026-08-01T12:15:51.201990Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T121551081167Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T121551081167Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0013477e7072c972201574e5a7d25d82b1748866 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T121551081167Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d04a0fd19e00b292cd2b1540df01eb30b9bcd8f42f147b03a90f1e0a2a71eef8 +size 6170089 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T121552070599Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T121552070599Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70ae9b0f4df506bdc2cb3f7f6573048496a17aba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T121552070599Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.070139Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:51.219354Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T121552070599Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.075508Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T121552070599Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T121552070599Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff3d257ca5fd567ee282a5b8ea9f9408149ddf8f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T121552070599Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d1dfc173b940c8d887aa8068820f66a6aae7185bd82610b525315ecb3db4095 +size 164216 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T121552941902Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T121552941902Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fcb5a5e6f48c269233752df733d2ac6710ff71c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T121552941902Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.941498Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.076403Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T121552941902Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.946445Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T121552941902Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T121552941902Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb96a98ae66f50f6642a639c1c313ab29ac4148f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T121552941902Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e46c5a5f23b0a945d34f1789eebc7a668bfef0bf95add4f548015d5ff3071fc +size 164216 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T121553783555Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T121553783555Z.json new file mode 100644 index 0000000000000000000000000000000000000000..870756e5bb4327cb2fe7798cef63d5cafe5aef56 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T121553783555Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:53.783151Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:52.947318Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T121553783555Z.pickle", + "timestamp_utc": "2026-08-01T12:15:53.787755Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T121553783555Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T121553783555Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f70e62e60e4d9c1c3d976e5846a6bc1a10fe3a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T121553783555Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0c488c01cefcadd1459f0358f7078a4c64aed53d93d56d481b2c57180f7fcbd +size 164216 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T121554625070Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T121554625070Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c65d1e67ea49d9ed2e1016f65d99d3576ff171f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T121554625070Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:54.624657Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:53.788611Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T121554625070Z.pickle", + "timestamp_utc": "2026-08-01T12:15:54.629376Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T121554625070Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T121554625070Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ecb7fef654266cd9a96d40e643780bb27772b79c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T121554625070Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d6cf301bdb1dd533d74ecd0af32221d0f8f5d5533c87daf3afb882ad433b551 +size 164216 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T121555467728Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T121555467728Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c38bdaf2c376c0cc716ab2be714bd960dc3d83fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T121555467728Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474102784, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:55.467276Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474102784, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:54.630253Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 138305077248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138305077248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 634, + "segment_count_current_delta": 0, + "segment_count_peak": 634, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T121555467728Z.pickle", + "timestamp_utc": "2026-08-01T12:15:55.472423Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T121555467728Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T121555467728Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..00fd3e7fe2d5b10db5d0328180cddffdb426a10d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T121555467728Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07f602daba6932da5ab9066eae19e975f8f974b79fc14a4dac78fef3a6b3b019 +size 164216 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T121647969491Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T121647969491Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ec22ac28188965df897b1a356ae6a6db245830a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T121647969491Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822272, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 123958148096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822272, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 61216822272, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 8849026048, + "inactive_split_bytes_peak": 15984751616, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 138328145920, + "reserved_bytes_peak": 138328145920, + "segment_count_current": 636, + "segment_count_peak": 636, + "timestamp_utc": "2026-08-01T12:16:47.969020Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822272, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 105813039104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474102784, + "active_bytes_peak": 123958148096, + "allocated_bytes_current": 72474102784, + "allocated_bytes_peak": 105813039104, + "device": 6, + "inactive_split_bytes_current": 12777223168, + "inactive_split_bytes_peak": 15984751616, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 138305077248, + "reserved_bytes_peak": 138305077248, + "segment_count_current": 634, + "segment_count_peak": 634, + "timestamp_utc": "2026-08-01T12:15:55.473272Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8849026048, + "inactive_split_bytes_current_delta": -3928197120, + "inactive_split_bytes_peak": 15984751616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 138328145920, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 138328145920, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 636, + "segment_count_current_delta": 2, + "segment_count_peak": 636, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T121647969491Z.pickle", + "timestamp_utc": "2026-08-01T12:16:48.049176Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T121647969491Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T121647969491Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14c3b782338e8813eac96ca83ecae75d50361a1c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T121647969491Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d14f1dbd020306c69f1a932605d09e1b9c7cf8a58131eab0777cd63f07d6632 +size 3959499 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T121546235684Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T121546235684Z.json new file mode 100644 index 0000000000000000000000000000000000000000..823d6d1e6ac3f04b7b217d3fe287de63f571b072 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T121546235684Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059840, + "active_bytes_current_delta": 37225604608, + "active_bytes_peak": 82164424704, + "active_bytes_peak_delta": 46698088448, + "after": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 82164424704, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 82164424704, + "device": 7, + "inactive_split_bytes_current": 7001932288, + "inactive_split_bytes_peak": 7845480448, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 94218747904, + "reserved_bytes_peak": 94218747904, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.235105Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059840, + "allocated_bytes_current_delta": 37225604608, + "allocated_bytes_peak": 82164424704, + "allocated_bytes_peak_delta": 46698088448, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:14:37.407735Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7001932288, + "inactive_split_bytes_current_delta": 6869113344, + "inactive_split_bytes_peak": 7845480448, + "inactive_split_bytes_peak_delta": 7685094400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 94218747904, + "reserved_bytes_current_delta": 54974742528, + "reserved_bytes_peak": 94218747904, + "reserved_bytes_peak_delta": 54974742528, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T121546235684Z.pickle", + "timestamp_utc": "2026-08-01T12:15:46.352494Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T121546235684Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T121546235684Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b1f5db59a4009d1f7ae772a123f89a491e7a4f21 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T121546235684Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7b865f6bbc658d9732179ca39ffee6fa8dfa4eebc30779add907d3fda433b54 +size 5925016 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T121547334008Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T121547334008Z.json new file mode 100644 index 0000000000000000000000000000000000000000..98d11884026a99d795ac7b2296285e6f7838268c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T121547334008Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105804444160, + "active_bytes_peak_delta": 23640019456, + "after": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 105804444160, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 105804444160, + "device": 7, + "inactive_split_bytes_current": 6773342720, + "inactive_split_bytes_peak": 7845480448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107095261184, + "reserved_bytes_peak": 107095261184, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T12:15:47.333612Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105804444160, + "allocated_bytes_peak_delta": 23640019456, + "before": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 82164424704, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 82164424704, + "device": 7, + "inactive_split_bytes_current": 7001932288, + "inactive_split_bytes_peak": 7845480448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 94218747904, + "reserved_bytes_peak": 94218747904, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:15:46.371666Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6773342720, + "inactive_split_bytes_current_delta": -228589568, + "inactive_split_bytes_peak": 7845480448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107095261184, + "reserved_bytes_current_delta": 12876513280, + "reserved_bytes_peak": 107095261184, + "reserved_bytes_peak_delta": 12876513280, + "segment_count_current": 480, + "segment_count_current_delta": 59, + "segment_count_peak": 480, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T121547334008Z.pickle", + "timestamp_utc": "2026-08-01T12:15:47.445777Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T121547334008Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T121547334008Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6832c92114c83a8d7460c83afac3b270cf8ca003 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T121547334008Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09d49413972243f3a25d6e3a18fe8a3f33410f1640da163e9d5ea9fbd16df585 +size 5883466 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T121548417524Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T121548417524Z.json new file mode 100644 index 0000000000000000000000000000000000000000..420648d00460afef7f8f88c795d2c84d229e9e9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T121548417524Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457059840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105804444160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 105804444160, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 105804444160, + "device": 7, + "inactive_split_bytes_current": 6773342720, + "inactive_split_bytes_peak": 7845480448, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107565023232, + "reserved_bytes_peak": 107565023232, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T12:15:48.417118Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457059840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105804444160, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 105804444160, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 105804444160, + "device": 7, + "inactive_split_bytes_current": 6773342720, + "inactive_split_bytes_peak": 7845480448, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107095261184, + "reserved_bytes_peak": 107095261184, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T12:15:47.461750Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6773342720, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7845480448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107565023232, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107565023232, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 481, + "segment_count_current_delta": 1, + "segment_count_peak": 481, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T121548417524Z.pickle", + "timestamp_utc": "2026-08-01T12:15:48.525272Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T121548417524Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T121548417524Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ff0bec8ff5875ace8e03e31614eb0fd32ac37f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T121548417524Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ebe94244ab52596dd496b3e6d39428ae10a31bb002e5aeed18f12670018151 +size 5887719 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T121551081228Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T121551081228Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96d7a490c880c4a519b7cfbc9616bcc588b114ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T121551081228Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 123958148608, + "active_bytes_peak_delta": 18153704448, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_peak": 15984751104, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 138307174400, + "reserved_bytes_peak": 138307174400, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:15:51.080699Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105813039616, + "allocated_bytes_peak_delta": 8595456, + "before": { + "active_bytes_current": 72457059840, + "active_bytes_peak": 105804444160, + "allocated_bytes_current": 72457059840, + "allocated_bytes_peak": 105804444160, + "device": 7, + "inactive_split_bytes_current": 6773342720, + "inactive_split_bytes_peak": 7845480448, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107565023232, + "reserved_bytes_peak": 107565023232, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T12:15:48.540869Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_current_delta": 6003879936, + "inactive_split_bytes_peak": 15984751104, + "inactive_split_bytes_peak_delta": 8139270656, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 138307174400, + "reserved_bytes_current_delta": 30742151168, + "reserved_bytes_peak": 138307174400, + "reserved_bytes_peak_delta": 30742151168, + "segment_count_current": 635, + "segment_count_current_delta": 154, + "segment_count_peak": 635, + "segment_count_peak_delta": 154, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T121551081228Z.pickle", + "timestamp_utc": "2026-08-01T12:15:51.204606Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T121551081228Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T121551081228Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6bb3e186f5efc54bb2f22d07d46e07513197518 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T121551081228Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:614c641de2f1afeedf03cd08d298f5391e80892618088f75c92154286a20ed64 +size 6171896 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T121552070714Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T121552070714Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f5943f3be2c3de07e23fea4aada86386bc4b6f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T121552070714Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_peak": 15984751104, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 138307174400, + "reserved_bytes_peak": 138307174400, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:15:52.070249Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_peak": 15984751104, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 138307174400, + "reserved_bytes_peak": 138307174400, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:15:51.221792Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751104, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 138307174400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138307174400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 635, + "segment_count_current_delta": 0, + "segment_count_peak": 635, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T121552070714Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.075686Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T121552070714Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T121552070714Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ad2bf342bf1b8043047f058b5334f2656e0895f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T121552070714Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33dcb26913c3037053aca1956a5fd79f604e001075af72fda1e37995e4ac0542 +size 166075 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T121552942100Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T121552942100Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7841b97ab79e5133657a7fa5e299dc423c9a8ee5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T121552942100Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_peak": 15984751104, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 138307174400, + "reserved_bytes_peak": 138307174400, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:15:52.941662Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_peak": 15984751104, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 138307174400, + "reserved_bytes_peak": 138307174400, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:15:52.076568Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751104, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 138307174400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138307174400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 635, + "segment_count_current_delta": 0, + "segment_count_peak": 635, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T121552942100Z.pickle", + "timestamp_utc": "2026-08-01T12:15:52.946686Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T121552942100Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T121552942100Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..359745ed3d7258910064d9984c3b97e61bb01e21 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T121552942100Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6978d76b9410b9d6ccf060f27b34fd221448cd52f6ca93172f3d11a44d5f34d7 +size 166075 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T121553783664Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T121553783664Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bc1ce98b2096f0d4c9d134a55fc90762630ea603 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T121553783664Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_peak": 15984751104, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 138307174400, + "reserved_bytes_peak": 138307174400, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:15:53.783252Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_peak": 15984751104, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 138307174400, + "reserved_bytes_peak": 138307174400, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:15:52.947587Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751104, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 138307174400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138307174400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 635, + "segment_count_current_delta": 0, + "segment_count_peak": 635, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T121553783664Z.pickle", + "timestamp_utc": "2026-08-01T12:15:53.787975Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T121553783664Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T121553783664Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf348f1362cad1b7bf51b390f7a78cf6c523a70a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T121553783664Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa1ed185788a837cc671211b8322d0514cbd1e31b60f37ae33bf0d3b72732c07 +size 166075 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T121554625172Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T121554625172Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9983062b7045eb60e5bafaa22c0a839faf07197a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T121554625172Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_peak": 15984751104, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 138307174400, + "reserved_bytes_peak": 138307174400, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:15:54.624758Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_peak": 15984751104, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 138307174400, + "reserved_bytes_peak": 138307174400, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:15:53.788853Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751104, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 138307174400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138307174400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 635, + "segment_count_current_delta": 0, + "segment_count_peak": 635, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T121554625172Z.pickle", + "timestamp_utc": "2026-08-01T12:15:54.629693Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T121554625172Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T121554625172Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..65c7efa639069b12f83337372daa79502ab767b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T121554625172Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deab41d395a79a922155d4bb778645291021ee636ab2b29e94482c45b528cecb +size 166075 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T121555467756Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T121555467756Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ba95276e4f56afc2c94972d65d4457eb3a0402ef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T121555467756Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72474103296, + "active_bytes_current_delta": 0, + "active_bytes_peak": 123958148608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_peak": 15984751104, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 138307174400, + "reserved_bytes_peak": 138307174400, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:15:55.467348Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474103296, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813039616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_peak": 15984751104, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 138307174400, + "reserved_bytes_peak": 138307174400, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:15:54.630581Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15984751104, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 138307174400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 138307174400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 635, + "segment_count_current_delta": 0, + "segment_count_peak": 635, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T121555467756Z.pickle", + "timestamp_utc": "2026-08-01T12:15:55.472091Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T121555467756Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T121555467756Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f03fb2bc8d8a9e86bb13385cece7bb525c5a3779 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T121555467756Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec8a97ae6ff17ee143de8a31114b6ae111fa00543342e8d04926784f41b0b7e0 +size 166075 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T121647969489Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T121647969489Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bad3c76af0f8bd2c4737ee3370b8d5bfa7778cc9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T121647969489Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61216822784, + "active_bytes_current_delta": -11257280512, + "active_bytes_peak": 123958148608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61216822784, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 61216822784, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 8849025536, + "inactive_split_bytes_peak": 15984751104, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 138330243072, + "reserved_bytes_peak": 138330243072, + "segment_count_current": 637, + "segment_count_peak": 637, + "timestamp_utc": "2026-08-01T12:16:47.969019Z", + "world_size": 8 + }, + "allocated_bytes_current": 61216822784, + "allocated_bytes_current_delta": -11257280512, + "allocated_bytes_peak": 105813039616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72474103296, + "active_bytes_peak": 123958148608, + "allocated_bytes_current": 72474103296, + "allocated_bytes_peak": 105813039616, + "device": 7, + "inactive_split_bytes_current": 12777222656, + "inactive_split_bytes_peak": 15984751104, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 138307174400, + "reserved_bytes_peak": 138307174400, + "segment_count_current": 635, + "segment_count_peak": 635, + "timestamp_utc": "2026-08-01T12:15:55.472946Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8849025536, + "inactive_split_bytes_current_delta": -3928197120, + "inactive_split_bytes_peak": 15984751104, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 138330243072, + "reserved_bytes_current_delta": 23068672, + "reserved_bytes_peak": 138330243072, + "reserved_bytes_peak_delta": 23068672, + "segment_count_current": 637, + "segment_count_current_delta": 2, + "segment_count_peak": 637, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T121647969489Z.pickle", + "timestamp_utc": "2026-08-01T12:16:48.048137Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T121647969489Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T121647969489Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2abadf86228b037b0a6af74bd877cbdcdd5b5d52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T121647969489Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f41bb0354752bdd31f98d5f805ad8a608de76fec120a450137a50c01dd55706 +size 3961322 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..bf08b78aac1c26be050de309a5e01a9c18d40c5a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 2048 +capacity: 1.50 +expert_dispatch_overlap: on +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6010 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 236 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T121947731868Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T121947731868Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bacde1891221e2a572dab07a644c272429c5ce49 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T121947731868Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871477248, + "active_bytes_current_delta": 31640022016, + "active_bytes_peak": 82183332864, + "active_bytes_peak_delta": 46716996608, + "after": { + "active_bytes_current": 66871477248, + "active_bytes_peak": 82183332864, + "allocated_bytes_current": 66871477248, + "allocated_bytes_peak": 82183332864, + "device": 0, + "inactive_split_bytes_current": 7157988352, + "inactive_split_bytes_peak": 7157989376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98631155712, + "reserved_bytes_peak": 98631155712, + "segment_count_current": 504, + "segment_count_peak": 504, + "timestamp_utc": "2026-08-01T12:19:47.731397Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871477248, + "allocated_bytes_current_delta": 31640022016, + "allocated_bytes_peak": 82183332864, + "allocated_bytes_peak_delta": 46716996608, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:18:38.035256Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7157988352, + "inactive_split_bytes_current_delta": 7023072256, + "inactive_split_bytes_peak": 7157989376, + "inactive_split_bytes_peak_delta": 6997357568, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98631155712, + "reserved_bytes_current_delta": 59385053184, + "reserved_bytes_peak": 98631155712, + "reserved_bytes_peak_delta": 59385053184, + "segment_count_current": 504, + "segment_count_current_delta": 432, + "segment_count_peak": 504, + "segment_count_peak_delta": 432, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T121947731868Z.pickle", + "timestamp_utc": "2026-08-01T12:19:47.849762Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T121947731868Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T121947731868Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bec14dc945b8800eb820f31cded3f295502de783 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T121947731868Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2705025ef62e06669c7ed0abc96d4d293cf3f355b39e5f3e6122cb1d7aa8b6b +size 5394399 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T121948700822Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T121948700822Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67699a6c22350d31a00b3d4cc40bb42ecbd502ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T121948700822Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871477248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105822312960, + "active_bytes_peak_delta": 23638980096, + "after": { + "active_bytes_current": 66871477248, + "active_bytes_peak": 105822312960, + "allocated_bytes_current": 66871477248, + "allocated_bytes_peak": 105822312960, + "device": 0, + "inactive_split_bytes_current": 7178959872, + "inactive_split_bytes_peak": 7413840896, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108680708096, + "reserved_bytes_peak": 108680708096, + "segment_count_current": 542, + "segment_count_peak": 542, + "timestamp_utc": "2026-08-01T12:19:48.700413Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871477248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105822312960, + "allocated_bytes_peak_delta": 23638980096, + "before": { + "active_bytes_current": 66871477248, + "active_bytes_peak": 82183332864, + "allocated_bytes_current": 66871477248, + "allocated_bytes_peak": 82183332864, + "device": 0, + "inactive_split_bytes_current": 7157988352, + "inactive_split_bytes_peak": 7157989376, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98631155712, + "reserved_bytes_peak": 98631155712, + "segment_count_current": 504, + "segment_count_peak": 504, + "timestamp_utc": "2026-08-01T12:19:47.869670Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7178959872, + "inactive_split_bytes_current_delta": 20971520, + "inactive_split_bytes_peak": 7413840896, + "inactive_split_bytes_peak_delta": 255851520, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108680708096, + "reserved_bytes_current_delta": 10049552384, + "reserved_bytes_peak": 108680708096, + "reserved_bytes_peak_delta": 10049552384, + "segment_count_current": 542, + "segment_count_current_delta": 38, + "segment_count_peak": 542, + "segment_count_peak_delta": 38, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T121948700822Z.pickle", + "timestamp_utc": "2026-08-01T12:19:48.807158Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T121948700822Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T121948700822Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca1476bcb552f135b455277e8e40d9a995cf05d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T121948700822Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f8a12c96fc3ee4c7783944a18424113ac6f6ceef597a8f8c1db29dee79c9a82 +size 5581869 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T121949663049Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T121949663049Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5d4c42b72147c214f5255d8dbca0cca659de0222 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T121949663049Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871477248, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105822312960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871477248, + "active_bytes_peak": 105822312960, + "allocated_bytes_current": 66871477248, + "allocated_bytes_peak": 105822312960, + "device": 0, + "inactive_split_bytes_current": 7176862720, + "inactive_split_bytes_peak": 8070759424, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109150470144, + "reserved_bytes_peak": 109150470144, + "segment_count_current": 544, + "segment_count_peak": 544, + "timestamp_utc": "2026-08-01T12:19:49.662639Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871477248, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105822312960, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871477248, + "active_bytes_peak": 105822312960, + "allocated_bytes_current": 66871477248, + "allocated_bytes_peak": 105822312960, + "device": 0, + "inactive_split_bytes_current": 7178959872, + "inactive_split_bytes_peak": 7413840896, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108680708096, + "reserved_bytes_peak": 108680708096, + "segment_count_current": 542, + "segment_count_peak": 542, + "timestamp_utc": "2026-08-01T12:19:48.822264Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7176862720, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8070759424, + "inactive_split_bytes_peak_delta": 656918528, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109150470144, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 109150470144, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 544, + "segment_count_current_delta": 2, + "segment_count_peak": 544, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T121949663049Z.pickle", + "timestamp_utc": "2026-08-01T12:19:49.766729Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T121949663049Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T121949663049Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..22121baaaa0a29424f33ce8342b2f167edba1843 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T121949663049Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef19e08537c509637e51e89b7593af525dbb49bb35bc0905603c81810d1eaa0b +size 5580128 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T121952281533Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T121952281533Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0dbf6b56e5d26a767ce3d32cff33ffd8f412385c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T121952281533Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888520704, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 157811140608, + "active_bytes_peak_delta": 51988827648, + "after": { + "active_bytes_current": 66888520704, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66888520704, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_peak": 10881535488, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_peak": 172253773824, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:52.280997Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888520704, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105836348928, + "allocated_bytes_peak_delta": 14035968, + "before": { + "active_bytes_current": 66871477248, + "active_bytes_peak": 105822312960, + "allocated_bytes_current": 66871477248, + "allocated_bytes_peak": 105822312960, + "device": 0, + "inactive_split_bytes_current": 7176862720, + "inactive_split_bytes_peak": 8070759424, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109150470144, + "reserved_bytes_peak": 109150470144, + "segment_count_current": 544, + "segment_count_peak": 544, + "timestamp_utc": "2026-08-01T12:19:49.779701Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_current_delta": -121901056, + "inactive_split_bytes_peak": 10881535488, + "inactive_split_bytes_peak_delta": 2810776064, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_current_delta": 63103303680, + "reserved_bytes_peak": 172253773824, + "reserved_bytes_peak_delta": 63103303680, + "segment_count_current": 897, + "segment_count_current_delta": 353, + "segment_count_peak": 897, + "segment_count_peak_delta": 353, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T121952281533Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.401202Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T121952281533Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T121952281533Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c54cc7e38ef481e925f9f89d72d548215962c250 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T121952281533Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:161a52b2c48c5c6227f40ca202b1233b518851717d0aea68619df1c564ea9760 +size 6584289 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T121952972710Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T121952972710Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bae01900fab8a09af248fddeb973071d588fedfd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T121952972710Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888520704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157811140608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888520704, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66888520704, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_peak": 10881535488, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_peak": 172253773824, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:52.972249Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888520704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836348928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888520704, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66888520704, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_peak": 10881535488, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_peak": 172253773824, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:52.418341Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10881535488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172253773824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 897, + "segment_count_current_delta": 0, + "segment_count_peak": 897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T121952972710Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.979585Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T121952972710Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T121952972710Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8cd58c10cb5cabc1763acf0511ce9e132e619dcd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T121952972710Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdb7c8aa5c29188fb8438613b9313b385d2cf0ad17634e6a8eb6f71eebddd391 +size 262813 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T121953540728Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T121953540728Z.json new file mode 100644 index 0000000000000000000000000000000000000000..020696e11b95207ea8703bd3a3aae4f4a77d23ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T121953540728Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888520704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157811140608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888520704, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66888520704, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_peak": 10881535488, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_peak": 172253773824, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:53.540338Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888520704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836348928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888520704, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66888520704, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_peak": 10881535488, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_peak": 172253773824, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:52.980551Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10881535488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172253773824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 897, + "segment_count_current_delta": 0, + "segment_count_peak": 897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T121953540728Z.pickle", + "timestamp_utc": "2026-08-01T12:19:53.547154Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T121953540728Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T121953540728Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0d7fdc4e1d1f7526dbb1fa90e32625f34d60784e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T121953540728Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc238bf7c2dd14e9002b29c5e3ae43ef64e8c66f57b0395ebd70af0f23ac32e8 +size 262813 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T121954101932Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T121954101932Z.json new file mode 100644 index 0000000000000000000000000000000000000000..558050658c6f64880e68c9e671646c6e0b2600d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T121954101932Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888520704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157811140608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888520704, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66888520704, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_peak": 10881535488, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_peak": 172253773824, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:54.101539Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888520704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836348928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888520704, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66888520704, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_peak": 10881535488, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_peak": 172253773824, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:53.548102Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10881535488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172253773824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 897, + "segment_count_current_delta": 0, + "segment_count_peak": 897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T121954101932Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.108014Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T121954101932Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T121954101932Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cda20529f2b3f58f9446a56dd4ae88e2e4dc708d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T121954101932Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b240e2889e1db4d0d6444ef7cdbf956382fe42080fcd6979b81a4af5b14cc6de +size 262813 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T121954662052Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T121954662052Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac256e4298063a43adc56a27c761d6aa4e252c3f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T121954662052Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888520704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157811140608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888520704, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66888520704, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_peak": 10881535488, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_peak": 172253773824, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:54.661666Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888520704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836348928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888520704, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66888520704, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_peak": 10881535488, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_peak": 172253773824, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:54.108964Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10881535488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172253773824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 897, + "segment_count_current_delta": 0, + "segment_count_peak": 897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T121954662052Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.668223Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T121954662052Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T121954662052Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ef9c1458fd232c6231182ab33ba0db9d5b08d3e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T121954662052Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f0eb82bf80428e2512df0cf6e828043f9cf2b6e6571515f1402fc4d2144ab1f +size 262813 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T121955220634Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T121955220634Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4b2ceacb45c64faf3f6074f2a84ae3fff6fbc07 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T121955220634Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888520704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157811140608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888520704, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66888520704, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_peak": 10881535488, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_peak": 172253773824, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:55.220247Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888520704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836348928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888520704, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66888520704, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_peak": 10881535488, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_peak": 172253773824, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:54.669189Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10881535488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172253773824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 897, + "segment_count_current_delta": 0, + "segment_count_peak": 897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T121955220634Z.pickle", + "timestamp_utc": "2026-08-01T12:19:55.226662Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T121955220634Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T121955220634Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..079df30c2324715ec74ec8931391ed734c9a7b14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T121955220634Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b576e61056fd7ed3b632918fe3afc35c2c1f4d51bd6b98b71ccc71d7c1adc1e +size 262813 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T122049955616Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T122049955616Z.json new file mode 100644 index 0000000000000000000000000000000000000000..56aa1531241574605fa254f1f656a88b6e50cb10 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T122049955616Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854886400, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 157811140608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854886400, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66854886400, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7034070016, + "inactive_split_bytes_peak": 10881535488, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 174216708096, + "reserved_bytes_peak": 174216708096, + "segment_count_current": 906, + "segment_count_peak": 906, + "timestamp_utc": "2026-08-01T12:20:49.955127Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854886400, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105836348928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888520704, + "active_bytes_peak": 157811140608, + "allocated_bytes_current": 66888520704, + "allocated_bytes_peak": 105836348928, + "device": 0, + "inactive_split_bytes_current": 7054961664, + "inactive_split_bytes_peak": 10881535488, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 172253773824, + "reserved_bytes_peak": 172253773824, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:55.227604Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7034070016, + "inactive_split_bytes_current_delta": -20891648, + "inactive_split_bytes_peak": 10881535488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 174216708096, + "reserved_bytes_current_delta": 1962934272, + "reserved_bytes_peak": 174216708096, + "reserved_bytes_peak_delta": 1962934272, + "segment_count_current": 906, + "segment_count_current_delta": 9, + "segment_count_peak": 906, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T122049955616Z.pickle", + "timestamp_utc": "2026-08-01T12:20:50.043116Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T122049955616Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T122049955616Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b4ae07234beff23d9c29be0b2ba9234c82431c76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T122049955616Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85b4e1c994202ba10e93cb42581a144f5941345c01be9365ce8c44b8cada2a4b +size 4115059 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T121947731977Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T121947731977Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f80473dfe118d781d19037d7b713d8e98e46eb9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T121947731977Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 31639604224, + "active_bytes_peak": 82185110528, + "active_bytes_peak_delta": 46718774272, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 82185110528, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 82185110528, + "device": 1, + "inactive_split_bytes_current": 7340858368, + "inactive_split_bytes_peak": 7340859392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 98754887680, + "reserved_bytes_peak": 98754887680, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:19:47.731366Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 31639604224, + "allocated_bytes_peak": 82185110528, + "allocated_bytes_peak_delta": 46718774272, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:18:38.035748Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7340858368, + "inactive_split_bytes_current_delta": 7208039424, + "inactive_split_bytes_peak": 7340859392, + "inactive_split_bytes_peak_delta": 7180473344, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 98754887680, + "reserved_bytes_current_delta": 59510882304, + "reserved_bytes_peak": 98754887680, + "reserved_bytes_peak_delta": 59510882304, + "segment_count_current": 503, + "segment_count_current_delta": 432, + "segment_count_peak": 503, + "segment_count_peak_delta": 432, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T121947731977Z.pickle", + "timestamp_utc": "2026-08-01T12:19:47.845049Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T121947731977Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T121947731977Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c2e0b939fc03601d9b78a039096aa9d3c44ba003 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T121947731977Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d72115c2ff6cd052c8660ca5d80982d296a98679e844e5cb244a569b0dfe2d41 +size 5384273 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T121948700832Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T121948700832Z.json new file mode 100644 index 0000000000000000000000000000000000000000..599626ca1e2a27f23229536105c7bee6eebc16d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T121948700832Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105828798976, + "active_bytes_peak_delta": 23643688448, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828798976, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828798976, + "device": 1, + "inactive_split_bytes_current": 7351344128, + "inactive_split_bytes_peak": 7705680896, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 109093847040, + "reserved_bytes_peak": 109093847040, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T12:19:48.700430Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105828798976, + "allocated_bytes_peak_delta": 23643688448, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 82185110528, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 82185110528, + "device": 1, + "inactive_split_bytes_current": 7340858368, + "inactive_split_bytes_peak": 7340859392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 98754887680, + "reserved_bytes_peak": 98754887680, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:19:47.864538Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7351344128, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 7705680896, + "inactive_split_bytes_peak_delta": 364821504, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 109093847040, + "reserved_bytes_current_delta": 10338959360, + "reserved_bytes_peak": 109093847040, + "reserved_bytes_peak_delta": 10338959360, + "segment_count_current": 538, + "segment_count_current_delta": 35, + "segment_count_peak": 538, + "segment_count_peak_delta": 35, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T121948700832Z.pickle", + "timestamp_utc": "2026-08-01T12:19:48.812675Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T121948700832Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T121948700832Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a4299b1c0822cec6d18c10cecad2ccff4ee3cfa8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T121948700832Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27e6872c252f9a9aab2602982eaf3409a8f45bcaeb954b318ea0f3038dc3996e +size 5584787 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T121949663063Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T121949663063Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b7d64fba14bc0387fd220c44dc6dddf03b681385 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T121949663063Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105828798976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828798976, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828798976, + "device": 1, + "inactive_split_bytes_current": 7122754560, + "inactive_split_bytes_peak": 8316796928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 109093847040, + "reserved_bytes_peak": 109093847040, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T12:19:49.662664Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105828798976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828798976, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828798976, + "device": 1, + "inactive_split_bytes_current": 7351344128, + "inactive_split_bytes_peak": 7705680896, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 109093847040, + "reserved_bytes_peak": 109093847040, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T12:19:48.828681Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7122754560, + "inactive_split_bytes_current_delta": -228589568, + "inactive_split_bytes_peak": 8316796928, + "inactive_split_bytes_peak_delta": 611116032, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 109093847040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109093847040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T121949663063Z.pickle", + "timestamp_utc": "2026-08-01T12:19:49.772098Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T121949663063Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T121949663063Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..310b7981a76c09a89ea0353103059eb7702e100f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T121949663063Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1810a70cee5d9ed5f5e8ed2091b6c1df842f1a8d79be268c92a4b00c94bf391f +size 5563761 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T121952281910Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T121952281910Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d23bf262643f5f207c68c91ac03966a9745c219 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T121952281910Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 51976025600, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_peak": 10982485504, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:52.281220Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 2544128, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828798976, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828798976, + "device": 1, + "inactive_split_bytes_current": 7122754560, + "inactive_split_bytes_peak": 8316796928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 109093847040, + "reserved_bytes_peak": 109093847040, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T12:19:49.789541Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_current_delta": 110882816, + "inactive_split_bytes_peak": 10982485504, + "inactive_split_bytes_peak_delta": 2665688576, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 172201345024, + "reserved_bytes_current_delta": 63107497984, + "reserved_bytes_peak": 172201345024, + "reserved_bytes_peak_delta": 63107497984, + "segment_count_current": 896, + "segment_count_current_delta": 358, + "segment_count_peak": 896, + "segment_count_peak_delta": 358, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T121952281910Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.402238Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T121952281910Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T121952281910Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a8c504dcc6fb61e9beac0d6d13e32e02319849ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T121952281910Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e214d5a1937ccc053ce466f020fa5949246ad2a8baf354b54dd5a7d8be9d488 +size 6583378 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T121952972750Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T121952972750Z.json new file mode 100644 index 0000000000000000000000000000000000000000..785da69d529e95677460d8ac061b38c44f19f660 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T121952972750Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_peak": 10982485504, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:52.972264Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_peak": 10982485504, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:52.418907Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10982485504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 172201345024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172201345024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 896, + "segment_count_current_delta": 0, + "segment_count_peak": 896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T121952972750Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.979694Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T121952972750Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T121952972750Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..67e3f584ce5539a11e9350194767b76b0a159628 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T121952972750Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83e1abb1d7bac55ec7af7d488ff574da0b7f10c6af142016df95265db4f2558b +size 261459 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T121953540727Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T121953540727Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9b85ea6e38a0b17d780ccd4f718ba30e2d5fdb35 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T121953540727Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_peak": 10982485504, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:53.540337Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_peak": 10982485504, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:52.980667Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10982485504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 172201345024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172201345024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 896, + "segment_count_current_delta": 0, + "segment_count_peak": 896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T121953540727Z.pickle", + "timestamp_utc": "2026-08-01T12:19:53.547228Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T121953540727Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T121953540727Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d4992b748818d27d1474c654a9cebb3e3caaba2d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T121953540727Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09695ac7893d8ceac8499df6a32b7654049963965b7cb23afb579e8dfa85d02d +size 261459 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T121954101955Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T121954101955Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf07f603c62999b5f2ce2069cd43ed3e2363eb59 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T121954101955Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_peak": 10982485504, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:54.101567Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_peak": 10982485504, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:53.548184Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10982485504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 172201345024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172201345024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 896, + "segment_count_current_delta": 0, + "segment_count_peak": 896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T121954101955Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.107945Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T121954101955Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T121954101955Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..79b0f976b3b8d7f9719667b1837ecb8b8b28b026 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T121954101955Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdd93c60de7a2774ace51e2695ac56efe22f7392c27173a885e25bfdffc82e46 +size 261459 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T121954662070Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T121954662070Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70baef1708030106b899115187790827f469eda4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T121954662070Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_peak": 10982485504, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:54.661678Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_peak": 10982485504, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:54.108898Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10982485504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 172201345024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172201345024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 896, + "segment_count_current_delta": 0, + "segment_count_peak": 896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T121954662070Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.668187Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T121954662070Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T121954662070Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d9e7bb52eaf5ac87f3286c51db3e29974a66bf2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T121954662070Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f30cb4eab3556af0f4ac40d10e8a7208f4f766b950b8dd53a65f26a7d9893047 +size 261459 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T121955220652Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T121955220652Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18bc87259fd210a8f9ea27a9500bee617c197a4b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T121955220652Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_peak": 10982485504, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:55.220269Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_peak": 10982485504, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:54.669138Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10982485504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 172201345024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172201345024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 896, + "segment_count_current_delta": 0, + "segment_count_peak": 896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T121955220652Z.pickle", + "timestamp_utc": "2026-08-01T12:19:55.226569Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T121955220652Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T121955220652Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ff9cd2ee34b99d71be215ced2cce6ff784d0c51 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T121955220652Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76be641cf8e5b634dd0953871d5ebf7b18547db6aec955474a2b6f9a04bf4fcf +size 261459 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T122049955630Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T122049955630Z.json new file mode 100644 index 0000000000000000000000000000000000000000..415f0b03f4ae80299f8d35d7ff00be4009cb3f95 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T122049955630Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854468608, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854468608, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66854468608, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 6977864704, + "inactive_split_bytes_peak": 10982485504, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 173224755200, + "reserved_bytes_peak": 173224755200, + "segment_count_current": 901, + "segment_count_peak": 901, + "timestamp_utc": "2026-08-01T12:20:49.955132Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854468608, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 1, + "inactive_split_bytes_current": 7233637376, + "inactive_split_bytes_peak": 10982485504, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:55.227523Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6977864704, + "inactive_split_bytes_current_delta": -255772672, + "inactive_split_bytes_peak": 10982485504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 173224755200, + "reserved_bytes_current_delta": 1023410176, + "reserved_bytes_peak": 173224755200, + "reserved_bytes_peak_delta": 1023410176, + "segment_count_current": 901, + "segment_count_current_delta": 5, + "segment_count_peak": 901, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T122049955630Z.pickle", + "timestamp_utc": "2026-08-01T12:20:50.044381Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T122049955630Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T122049955630Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a173e199c5092db90c37f0fc4f1b72986c7913a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T122049955630Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cc1095cd9a4170caa4e2c3cbe3c0dae2efdceda98f27379c88a204fb70124de +size 3934181 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T121947731982Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T121947731982Z.json new file mode 100644 index 0000000000000000000000000000000000000000..044320be6a8a46ac7136b3cb028e4e8eb59d8edb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T121947731982Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 31639604224, + "active_bytes_peak": 82186683392, + "active_bytes_peak_delta": 46720347136, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 82186683392, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 82186683392, + "device": 2, + "inactive_split_bytes_current": 7045159936, + "inactive_split_bytes_peak": 7045160960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98928951296, + "reserved_bytes_peak": 98928951296, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T12:19:47.731399Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 31639604224, + "allocated_bytes_peak": 82186683392, + "allocated_bytes_peak_delta": 46720347136, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:18:38.035688Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7045159936, + "inactive_split_bytes_current_delta": 6912340992, + "inactive_split_bytes_peak": 7045160960, + "inactive_split_bytes_peak_delta": 6884774912, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98928951296, + "reserved_bytes_current_delta": 59684945920, + "reserved_bytes_peak": 98928951296, + "reserved_bytes_peak_delta": 59684945920, + "segment_count_current": 505, + "segment_count_current_delta": 434, + "segment_count_peak": 505, + "segment_count_peak_delta": 434, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T121947731982Z.pickle", + "timestamp_utc": "2026-08-01T12:19:47.843066Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T121947731982Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T121947731982Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db6c954228de5bfb03f88bec9c314b8a76cede6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T121947731982Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d75f95d2bc51540f61353990d213ce5f8c385dbd58c3714e3b5b4a44470d81e +size 5384578 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T121948700827Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T121948700827Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e9e3e55a14a06338cd6a038be367cc5b73238680 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T121948700827Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105826300416, + "active_bytes_peak_delta": 23639617024, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105826300416, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105826300416, + "device": 2, + "inactive_split_bytes_current": 6824958976, + "inactive_split_bytes_peak": 7845143552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107795709952, + "reserved_bytes_peak": 107795709952, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T12:19:48.700419Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105826300416, + "allocated_bytes_peak_delta": 23639617024, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 82186683392, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 82186683392, + "device": 2, + "inactive_split_bytes_current": 7045159936, + "inactive_split_bytes_peak": 7045160960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98928951296, + "reserved_bytes_peak": 98928951296, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T12:19:47.862670Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6824958976, + "inactive_split_bytes_current_delta": -220200960, + "inactive_split_bytes_peak": 7845143552, + "inactive_split_bytes_peak_delta": 799982592, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107795709952, + "reserved_bytes_current_delta": 8866758656, + "reserved_bytes_peak": 107795709952, + "reserved_bytes_peak_delta": 8866758656, + "segment_count_current": 538, + "segment_count_current_delta": 33, + "segment_count_peak": 538, + "segment_count_peak_delta": 33, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T121948700827Z.pickle", + "timestamp_utc": "2026-08-01T12:19:48.805693Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T121948700827Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T121948700827Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de2d5762774e4ea49793e4e60d00fb1183d59183 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T121948700827Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7530162990cb751416374debb52f904480c67694dcb3ce01c4bae41cb5831c2 +size 5585262 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T121949663061Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T121949663061Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9f32bc83642b8903f07b50351381f9dd4a0ab22e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T121949663061Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105826300416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105826300416, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105826300416, + "device": 2, + "inactive_split_bytes_current": 6824958976, + "inactive_split_bytes_peak": 7845143552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109383254016, + "reserved_bytes_peak": 109383254016, + "segment_count_current": 542, + "segment_count_peak": 542, + "timestamp_utc": "2026-08-01T12:19:49.662665Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105826300416, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105826300416, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105826300416, + "device": 2, + "inactive_split_bytes_current": 6824958976, + "inactive_split_bytes_peak": 7845143552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107795709952, + "reserved_bytes_peak": 107795709952, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T12:19:48.821098Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6824958976, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7845143552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109383254016, + "reserved_bytes_current_delta": 1587544064, + "reserved_bytes_peak": 109383254016, + "reserved_bytes_peak_delta": 1587544064, + "segment_count_current": 542, + "segment_count_current_delta": 4, + "segment_count_peak": 542, + "segment_count_peak_delta": 4, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T121949663061Z.pickle", + "timestamp_utc": "2026-08-01T12:19:49.763846Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T121949663061Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T121949663061Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb850c67994558e426a3b83de32c4a4e638a8ae2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T121949663061Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:848ebb941c2013e74b6001cbc1e7745d2552c3250e7a3af707a6ef4ae5bea2c7 +size 5579860 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T121952281535Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T121952281535Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c068ce6e3376e726dcd94fdedd8828371b74d21b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T121952281535Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 157810853888, + "active_bytes_peak_delta": 51984553472, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_peak": 10659786240, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:52.281000Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105834227712, + "allocated_bytes_peak_delta": 7927296, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105826300416, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105826300416, + "device": 2, + "inactive_split_bytes_current": 6824958976, + "inactive_split_bytes_peak": 7845143552, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109383254016, + "reserved_bytes_peak": 109383254016, + "segment_count_current": 542, + "segment_count_peak": 542, + "timestamp_utc": "2026-08-01T12:19:49.777034Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_current_delta": 115077120, + "inactive_split_bytes_peak": 10659786240, + "inactive_split_bytes_peak_delta": 2814642688, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 171903549440, + "reserved_bytes_current_delta": 62520295424, + "reserved_bytes_peak": 171903549440, + "reserved_bytes_peak_delta": 62520295424, + "segment_count_current": 895, + "segment_count_current_delta": 353, + "segment_count_peak": 895, + "segment_count_peak_delta": 353, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T121952281535Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.397760Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T121952281535Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T121952281535Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..01a36bd144f3413473551b579e7ed3286e62acfd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T121952281535Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80d805c844e3476f06fede51424b9ef5f968cb78ca3c16a13c2fc5f752e74b24 +size 6585041 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T121952972761Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T121952972761Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e20cf9391ea5c4f9d44db7f1c18af8fd0b2eb7f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T121952972761Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810853888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_peak": 10659786240, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:52.972291Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834227712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_peak": 10659786240, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:52.412226Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10659786240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 171903549440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171903549440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 895, + "segment_count_current_delta": 0, + "segment_count_peak": 895, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T121952972761Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.979434Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T121952972761Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T121952972761Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f6a6099125201771e572155c871ad85d725638c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T121952972761Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deffd1b10d34a05ee61a221a5d524c96238f022a5f1f9c322d643be8a4c57867 +size 261401 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T121953540723Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T121953540723Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c2c6f4e754bce859ee4854008e096272ad8c54c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T121953540723Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810853888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_peak": 10659786240, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:53.540337Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834227712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_peak": 10659786240, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:52.980393Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10659786240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 171903549440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171903549440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 895, + "segment_count_current_delta": 0, + "segment_count_peak": 895, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T121953540723Z.pickle", + "timestamp_utc": "2026-08-01T12:19:53.546946Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T121953540723Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T121953540723Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9f66551491e1daf1df0f73ca58562e3fe9bc960 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T121953540723Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59f492b208d9ebf82ab4596abde7e4ade90293472f43b39a01eddf2a15c624f9 +size 261395 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T121954101912Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T121954101912Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c74ae7101bcf389c6c1b1a89f6eb4bafa53cf1ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T121954101912Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810853888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_peak": 10659786240, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:54.101538Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834227712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_peak": 10659786240, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:53.547880Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10659786240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 171903549440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171903549440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 895, + "segment_count_current_delta": 0, + "segment_count_peak": 895, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T121954101912Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.107768Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T121954101912Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T121954101912Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2a40a233d3b06cf4404e3c9fa41da72f2b74f65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T121954101912Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87f897cac7a097689f36a1f34d23277bdf69f7ffee715c4725ab0b7c20d7d60b +size 261401 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T121954662051Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T121954662051Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bfe008406c1f1b2de61f1235c21adbfcf070ab44 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T121954662051Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810853888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_peak": 10659786240, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:54.661660Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834227712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_peak": 10659786240, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:54.108708Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10659786240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 171903549440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171903549440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 895, + "segment_count_current_delta": 0, + "segment_count_peak": 895, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T121954662051Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.668071Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T121954662051Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T121954662051Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2673eca9b745f1b7f7f4060801a41d37681e4f67 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T121954662051Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18728717ce66220582420b640755dcbdeef5bfa2267e8097c9de648664939246 +size 261395 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T121955220636Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T121955220636Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e7a6cf10c4c614137a0401928fa5962afa9abb3e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T121955220636Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810853888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_peak": 10659786240, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:55.220255Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834227712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_peak": 10659786240, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:54.669046Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10659786240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 171903549440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171903549440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 895, + "segment_count_current_delta": 0, + "segment_count_peak": 895, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T121955220636Z.pickle", + "timestamp_utc": "2026-08-01T12:19:55.226451Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T121955220636Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T121955220636Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eb6207f21b63b29200428177b938b91a9c3aa575 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T121955220636Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb130e5a1d6bf5d4bc01b49918652d6ecd6dab2e81b4b997cf2660f06e278894 +size 261401 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T122049955624Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T122049955624Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4025eebc1cb71127e108461a12796a149a9d013 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T122049955624Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854468608, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 157810853888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854468608, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66854468608, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6684263424, + "inactive_split_bytes_peak": 10659786240, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 173631602688, + "reserved_bytes_peak": 173631602688, + "segment_count_current": 903, + "segment_count_peak": 903, + "timestamp_utc": "2026-08-01T12:20:49.955127Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854468608, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105834227712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810853888, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105834227712, + "device": 2, + "inactive_split_bytes_current": 6940036096, + "inactive_split_bytes_peak": 10659786240, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:55.227383Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6684263424, + "inactive_split_bytes_current_delta": -255772672, + "inactive_split_bytes_peak": 10659786240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 173631602688, + "reserved_bytes_current_delta": 1728053248, + "reserved_bytes_peak": 173631602688, + "reserved_bytes_peak_delta": 1728053248, + "segment_count_current": 903, + "segment_count_current_delta": 8, + "segment_count_peak": 903, + "segment_count_peak_delta": 8, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T122049955624Z.pickle", + "timestamp_utc": "2026-08-01T12:20:50.038286Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T122049955624Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T122049955624Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e7609cbc99e462bdf0caf7232979e9e20cf539e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T122049955624Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e575b8356bcda0d749e1434bcb5cf9f8d9ae66eef4ed80dc4601558576297972 +size 3932462 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T121947732944Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T121947732944Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a31e6cb81fb3609c62ee71e213ab4e1b3ace97e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T121947732944Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 31639604224, + "active_bytes_peak": 82186291200, + "active_bytes_peak_delta": 46719954944, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 82186291200, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 82186291200, + "device": 3, + "inactive_split_bytes_current": 7340858368, + "inactive_split_bytes_peak": 7340859392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 98754887680, + "reserved_bytes_peak": 98754887680, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:19:47.732276Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 31639604224, + "allocated_bytes_peak": 82186291200, + "allocated_bytes_peak_delta": 46719954944, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:18:38.035613Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7340858368, + "inactive_split_bytes_current_delta": 7208039424, + "inactive_split_bytes_peak": 7340859392, + "inactive_split_bytes_peak_delta": 7180473344, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 98754887680, + "reserved_bytes_current_delta": 59510882304, + "reserved_bytes_peak": 98754887680, + "reserved_bytes_peak_delta": 59510882304, + "segment_count_current": 503, + "segment_count_current_delta": 432, + "segment_count_peak": 503, + "segment_count_peak_delta": 432, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T121947732944Z.pickle", + "timestamp_utc": "2026-08-01T12:19:47.857819Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T121947732944Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T121947732944Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5f425d8f8d03c27ff5c5e867fa21d5f4bb6b9575 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T121947732944Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dffebaf9973cf5765029eccd55aed48fa7c931eadd94ee4eb0147778550077cd +size 5384272 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T121948700860Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T121948700860Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0e6bb94bd2c495c69228ceb950f5275716389992 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T121948700860Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105828901376, + "active_bytes_peak_delta": 23642610176, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828901376, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828901376, + "device": 3, + "inactive_split_bytes_current": 7120657408, + "inactive_split_bytes_peak": 7711608832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107690852352, + "reserved_bytes_peak": 107690852352, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T12:19:48.700446Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105828901376, + "allocated_bytes_peak_delta": 23642610176, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 82186291200, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 82186291200, + "device": 3, + "inactive_split_bytes_current": 7340858368, + "inactive_split_bytes_peak": 7340859392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 98754887680, + "reserved_bytes_peak": 98754887680, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:19:47.878893Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7120657408, + "inactive_split_bytes_current_delta": -220200960, + "inactive_split_bytes_peak": 7711608832, + "inactive_split_bytes_peak_delta": 370749440, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107690852352, + "reserved_bytes_current_delta": 8935964672, + "reserved_bytes_peak": 107690852352, + "reserved_bytes_peak_delta": 8935964672, + "segment_count_current": 533, + "segment_count_current_delta": 30, + "segment_count_peak": 533, + "segment_count_peak_delta": 30, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T121948700860Z.pickle", + "timestamp_utc": "2026-08-01T12:19:48.805582Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T121948700860Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T121948700860Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..79f600611691994099a83b6fd5c869c5a213c02d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T121948700860Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7e6271d1f9fa2fddeea6c86b2fc640d799608b0972504b46e0f76a204625193 +size 5583081 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T121949663344Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T121949663344Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7b7419e75940a132af33d297ac7a37a8e442dc5a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T121949663344Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105828901376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828901376, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828901376, + "device": 3, + "inactive_split_bytes_current": 7122754560, + "inactive_split_bytes_peak": 7770774528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108573753344, + "reserved_bytes_peak": 108573753344, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:19:49.662842Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105828901376, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828901376, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828901376, + "device": 3, + "inactive_split_bytes_current": 7120657408, + "inactive_split_bytes_peak": 7711608832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107690852352, + "reserved_bytes_peak": 107690852352, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T12:19:48.821215Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7122754560, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 7770774528, + "inactive_split_bytes_peak_delta": 59165696, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108573753344, + "reserved_bytes_current_delta": 882900992, + "reserved_bytes_peak": 108573753344, + "reserved_bytes_peak_delta": 882900992, + "segment_count_current": 535, + "segment_count_current_delta": 2, + "segment_count_peak": 535, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T121949663344Z.pickle", + "timestamp_utc": "2026-08-01T12:19:49.774521Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T121949663344Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T121949663344Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..521b804d95a19d269714c05f3876652bccbe5ae9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T121949663344Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3211b6ef3f3640153015509a2d31f7ad3e69dc790c95274fb6428051f9260e5 +size 5562370 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T121952281748Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T121952281748Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee85749119d6f22519a26127142a212e3b25b00c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T121952281748Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 51982083584, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_peak": 10744327680, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:52.281127Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 7685120, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828901376, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828901376, + "device": 3, + "inactive_split_bytes_current": 7122754560, + "inactive_split_bytes_peak": 7770774528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108573753344, + "reserved_bytes_peak": 108573753344, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:19:49.789685Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_current_delta": 112979968, + "inactive_split_bytes_peak": 10744327680, + "inactive_split_bytes_peak_delta": 2973553152, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 172201345024, + "reserved_bytes_current_delta": 63627591680, + "reserved_bytes_peak": 172201345024, + "reserved_bytes_peak_delta": 63627591680, + "segment_count_current": 896, + "segment_count_current_delta": 361, + "segment_count_peak": 896, + "segment_count_peak_delta": 361, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T121952281748Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.402974Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T121952281748Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T121952281748Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e69a5977b4a2548cbc31cabcb135bc2edd90d322 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T121952281748Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03d016b968e338a1d7bbf6a59d6f51b6b1156e917e550294f192dbd78712c845 +size 6582570 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T121952972912Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T121952972912Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6ee270abcfdcd86a985f0143de435e1d1e9b607 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T121952972912Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_peak": 10744327680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:52.972378Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_peak": 10744327680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:52.418784Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10744327680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 172201345024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172201345024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 896, + "segment_count_current_delta": 0, + "segment_count_peak": 896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T121952972912Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.980270Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T121952972912Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T121952972912Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5cb9795dbe17b7a48cb433dd0b3935c04c5543d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T121952972912Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01d661dd58327c20c91fb3bee1f3a2b9758082193eee8b9e84e55534afbce0d2 +size 260635 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T121953540852Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T121953540852Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ced77737bad9727079f2857e44dee9eebdd85553 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T121953540852Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_peak": 10744327680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:53.540448Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_peak": 10744327680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:52.981351Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10744327680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 172201345024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172201345024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 896, + "segment_count_current_delta": 0, + "segment_count_peak": 896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T121953540852Z.pickle", + "timestamp_utc": "2026-08-01T12:19:53.547350Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T121953540852Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T121953540852Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7b65225cc3b4314e96514fae5a123e3f2a3a6622 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T121953540852Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8913a8c2df341ea8a6024c3f4ca9242b71a89c59f08fbbb0aa2c840d783b9a69 +size 260635 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T121954102031Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T121954102031Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b8f154d11756a084523228fffdc0e770cb7b47ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T121954102031Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_peak": 10744327680, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:54.101636Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_peak": 10744327680, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:53.548337Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10744327680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 172201345024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172201345024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 896, + "segment_count_current_delta": 0, + "segment_count_peak": 896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T121954102031Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.108065Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T121954102031Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T121954102031Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a29edc7c1c419cf894f108fc376f6fb443b21f97 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T121954102031Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd8b25b447fe5fa71aa568cb60a3547c54b8655ef224dd9a8ac1132a70ca4e02 +size 260635 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T121954662149Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T121954662149Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7afacb1a231400a154490e9280c05e1296ec737c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T121954662149Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_peak": 10744327680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:54.661760Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_peak": 10744327680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:54.109037Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10744327680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 172201345024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172201345024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 896, + "segment_count_current_delta": 0, + "segment_count_peak": 896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T121954662149Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.668265Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T121954662149Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T121954662149Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d0deab5a51efcaf6e62d324f599b73f742ee548a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T121954662149Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ec65b89c725444b8e1a7ed0c7330aa388b5c200988013d706b2f2031a67820c +size 260635 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T121955220717Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T121955220717Z.json new file mode 100644 index 0000000000000000000000000000000000000000..498a6d81746c199026132855d4d6d143a458843e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T121955220717Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_peak": 10744327680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:55.220327Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_peak": 10744327680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:54.669234Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10744327680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 172201345024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172201345024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 896, + "segment_count_current_delta": 0, + "segment_count_peak": 896, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T121955220717Z.pickle", + "timestamp_utc": "2026-08-01T12:19:55.226698Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T121955220717Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T121955220717Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e9395a480320d4d2673a07268e4badd81e8f473c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T121955220717Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c89307737a695c5fe501b9154211b9fddc7953c3bcbed4c2c16427f0b4d3471a +size 260635 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T122049955704Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T122049955704Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c37579e63209a211d9dfc82affd5ff5fd3670055 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T122049955704Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854468608, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854468608, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66854468608, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 6979961856, + "inactive_split_bytes_peak": 10744327680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 173224755200, + "reserved_bytes_peak": 173224755200, + "segment_count_current": 901, + "segment_count_peak": 901, + "timestamp_utc": "2026-08-01T12:20:49.955209Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854468608, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 3, + "inactive_split_bytes_current": 7235734528, + "inactive_split_bytes_peak": 10744327680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 172201345024, + "reserved_bytes_peak": 172201345024, + "segment_count_current": 896, + "segment_count_peak": 896, + "timestamp_utc": "2026-08-01T12:19:55.227658Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6979961856, + "inactive_split_bytes_current_delta": -255772672, + "inactive_split_bytes_peak": 10744327680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 173224755200, + "reserved_bytes_current_delta": 1023410176, + "reserved_bytes_peak": 173224755200, + "reserved_bytes_peak_delta": 1023410176, + "segment_count_current": 901, + "segment_count_current_delta": 5, + "segment_count_peak": 901, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T122049955704Z.pickle", + "timestamp_utc": "2026-08-01T12:20:50.053968Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T122049955704Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T122049955704Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4d375d371e4e4c751d39e21e998106af59092268 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T122049955704Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de104ab445101cfcfc9a44159d673d88b69dacd637b6b7ce90f5971b34f88dd9 +size 3933316 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T121947731942Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T121947731942Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7fbcdc44ba01e05113b5a54ce038de82071baa96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T121947731942Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872657920, + "active_bytes_current_delta": 31640760320, + "active_bytes_peak": 82182490112, + "active_bytes_peak_delta": 46715711488, + "after": { + "active_bytes_current": 66872657920, + "active_bytes_peak": 82182490112, + "allocated_bytes_current": 66872657920, + "allocated_bytes_peak": 82182490112, + "device": 4, + "inactive_split_bytes_current": 7316191232, + "inactive_split_bytes_peak": 7316192256, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 98981380096, + "reserved_bytes_peak": 98981380096, + "segment_count_current": 504, + "segment_count_peak": 504, + "timestamp_utc": "2026-08-01T12:19:47.731478Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872657920, + "allocated_bytes_current_delta": 31640760320, + "allocated_bytes_peak": 82182490112, + "allocated_bytes_peak_delta": 46715711488, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:18:38.035255Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7316191232, + "inactive_split_bytes_current_delta": 7183814656, + "inactive_split_bytes_peak": 7316192256, + "inactive_split_bytes_peak_delta": 7162782720, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 98981380096, + "reserved_bytes_current_delta": 59737374720, + "reserved_bytes_peak": 98981380096, + "reserved_bytes_peak_delta": 59737374720, + "segment_count_current": 504, + "segment_count_current_delta": 433, + "segment_count_peak": 504, + "segment_count_peak_delta": 433, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T121947731942Z.pickle", + "timestamp_utc": "2026-08-01T12:19:47.845759Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T121947731942Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T121947731942Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b1f46a2887c3f12f25639f6be82c8e6a5d4d37d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T121947731942Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8664653652eb401964113ef2e9779a168bc8a40475c854b52e03f7e061212d67 +size 5393271 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T121948700821Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T121948700821Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8acf8f8999b68a55566c4bb526268157bb969e63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T121948700821Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872657920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105824801280, + "active_bytes_peak_delta": 23642311168, + "after": { + "active_bytes_current": 66872657920, + "active_bytes_peak": 105824801280, + "allocated_bytes_current": 66872657920, + "allocated_bytes_peak": 105824801280, + "device": 4, + "inactive_split_bytes_current": 7335065600, + "inactive_split_bytes_peak": 8185201664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108527616000, + "reserved_bytes_peak": 108527616000, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T12:19:48.700421Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872657920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105824801280, + "allocated_bytes_peak_delta": 23642311168, + "before": { + "active_bytes_current": 66872657920, + "active_bytes_peak": 82182490112, + "allocated_bytes_current": 66872657920, + "allocated_bytes_peak": 82182490112, + "device": 4, + "inactive_split_bytes_current": 7316191232, + "inactive_split_bytes_peak": 7316192256, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 98981380096, + "reserved_bytes_peak": 98981380096, + "segment_count_current": 504, + "segment_count_peak": 504, + "timestamp_utc": "2026-08-01T12:19:47.863587Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7335065600, + "inactive_split_bytes_current_delta": 18874368, + "inactive_split_bytes_peak": 8185201664, + "inactive_split_bytes_peak_delta": 869009408, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108527616000, + "reserved_bytes_current_delta": 9546235904, + "reserved_bytes_peak": 108527616000, + "reserved_bytes_peak_delta": 9546235904, + "segment_count_current": 537, + "segment_count_current_delta": 33, + "segment_count_peak": 537, + "segment_count_peak_delta": 33, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T121948700821Z.pickle", + "timestamp_utc": "2026-08-01T12:19:48.802703Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T121948700821Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T121948700821Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0942722815b3f1929dab29752c51942a2da4ad0c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T121948700821Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:660a8b99781e30c3361e55eb459760ab5d1b346f4e052acd5358f86fef8167ab +size 5592864 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T121949663053Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T121949663053Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ed28b7388bf56a07df6315e4b0d57b5ade4705f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T121949663053Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66872657920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105824801280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66872657920, + "active_bytes_peak": 105824801280, + "allocated_bytes_current": 66872657920, + "allocated_bytes_peak": 105824801280, + "device": 4, + "inactive_split_bytes_current": 7335065600, + "inactive_split_bytes_peak": 8185201664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108997378048, + "reserved_bytes_peak": 108997378048, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T12:19:49.662656Z", + "world_size": 8 + }, + "allocated_bytes_current": 66872657920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105824801280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66872657920, + "active_bytes_peak": 105824801280, + "allocated_bytes_current": 66872657920, + "allocated_bytes_peak": 105824801280, + "device": 4, + "inactive_split_bytes_current": 7335065600, + "inactive_split_bytes_peak": 8185201664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108527616000, + "reserved_bytes_peak": 108527616000, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T12:19:48.816033Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7335065600, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8185201664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108997378048, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 108997378048, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 539, + "segment_count_current_delta": 2, + "segment_count_peak": 539, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T121949663053Z.pickle", + "timestamp_utc": "2026-08-01T12:19:49.768109Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T121949663053Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T121949663053Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6844025d853f6455b6517bec3178ff4fb6841655 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T121949663053Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0591b66caecd44d8371db780fb69025836f362adb697c4bcc2dafda7c718d7ce +size 5584599 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T121952281557Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T121952281557Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5d3b4ea11f35b79748c04263807ce3c604d23303 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T121952281557Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889701376, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 157809437696, + "active_bytes_peak_delta": 51984636416, + "after": { + "active_bytes_current": 66889701376, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66889701376, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_peak": 10756893696, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 172415254528, + "reserved_bytes_peak": 172415254528, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:52.281007Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889701376, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105834514944, + "allocated_bytes_peak_delta": 9713664, + "before": { + "active_bytes_current": 66872657920, + "active_bytes_peak": 105824801280, + "allocated_bytes_current": 66872657920, + "allocated_bytes_peak": 105824801280, + "device": 4, + "inactive_split_bytes_current": 7335065600, + "inactive_split_bytes_peak": 8185201664, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108997378048, + "reserved_bytes_peak": 108997378048, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T12:19:49.781762Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_current_delta": -121901056, + "inactive_split_bytes_peak": 10756893696, + "inactive_split_bytes_peak_delta": 2571692032, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 172415254528, + "reserved_bytes_current_delta": 63417876480, + "reserved_bytes_peak": 172415254528, + "reserved_bytes_peak_delta": 63417876480, + "segment_count_current": 894, + "segment_count_current_delta": 355, + "segment_count_peak": 894, + "segment_count_peak_delta": 355, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T121952281557Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.401637Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T121952281557Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T121952281557Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cd294abe73d3c2f1bb372d14cff25b3f019737df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T121952281557Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4da52937359a6686d97496a8a4f4a07b73c9221b43fdc914f758be28d87a2676 +size 6591455 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T121952972705Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T121952972705Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d2146c73f824d62e31ddc8757c5f6981e978d3a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T121952972705Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889701376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157809437696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889701376, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66889701376, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_peak": 10756893696, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 172415254528, + "reserved_bytes_peak": 172415254528, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:52.972253Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889701376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834514944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889701376, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66889701376, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_peak": 10756893696, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 172415254528, + "reserved_bytes_peak": 172415254528, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:52.419217Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10756893696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 172415254528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172415254528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 894, + "segment_count_current_delta": 0, + "segment_count_peak": 894, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T121952972705Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.979552Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T121952972705Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T121952972705Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2c105e30961b89c5ed029c1460ced1eced46a2b4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T121952972705Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bbd091aeda5f8dc0eb3e147a18444ef61ed8399e8611edcf3b9c0e4a80b20bf +size 267698 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T121953540793Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T121953540793Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e673557771405cf3feb1a0d4788f0abcf9bb148 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T121953540793Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889701376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157809437696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889701376, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66889701376, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_peak": 10756893696, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 172415254528, + "reserved_bytes_peak": 172415254528, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:53.540326Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889701376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834514944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889701376, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66889701376, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_peak": 10756893696, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 172415254528, + "reserved_bytes_peak": 172415254528, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:52.980510Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10756893696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 172415254528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172415254528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 894, + "segment_count_current_delta": 0, + "segment_count_peak": 894, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T121953540793Z.pickle", + "timestamp_utc": "2026-08-01T12:19:53.547222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T121953540793Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T121953540793Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..07bc5b66249484dac8239e55e9b1c1026e238dc3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T121953540793Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:310d91395e59ff0bc9e11101cfdcc2b93530518a9aa9fe5a324b94188706c9da +size 267698 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T121954101938Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T121954101938Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96897fc4e8060c2d8375c1519b6e950cbb30ee7f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T121954101938Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889701376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157809437696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889701376, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66889701376, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_peak": 10756893696, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 172415254528, + "reserved_bytes_peak": 172415254528, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:54.101545Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889701376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834514944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889701376, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66889701376, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_peak": 10756893696, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 172415254528, + "reserved_bytes_peak": 172415254528, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:53.548144Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10756893696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 172415254528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172415254528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 894, + "segment_count_current_delta": 0, + "segment_count_peak": 894, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T121954101938Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.108107Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T121954101938Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T121954101938Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bceee0583639af55beb2efbbab4c3c6377497213 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T121954101938Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25bdb9392a7bf4020db1711bbaa1996f0f8d022b8eb63d5ebbeac5a4c753d2e7 +size 267698 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T121954662092Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T121954662092Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f0a69b4c10db76f48898263d1f3fe0b290ec1d9e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T121954662092Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889701376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157809437696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889701376, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66889701376, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_peak": 10756893696, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 172415254528, + "reserved_bytes_peak": 172415254528, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:54.661699Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889701376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834514944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889701376, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66889701376, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_peak": 10756893696, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 172415254528, + "reserved_bytes_peak": 172415254528, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:54.109054Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10756893696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 172415254528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172415254528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 894, + "segment_count_current_delta": 0, + "segment_count_peak": 894, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T121954662092Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.668353Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T121954662092Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T121954662092Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94bf949a0f6f87c4c3d32651dc1bb598c0c33439 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T121954662092Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9dbf0dc944c1a38d4b4ca0c66daeac05bb7a2a1041d60bf12f6fe1db1782751 +size 267698 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T121955220639Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T121955220639Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9ce2c7ec807d6db73d0830390d35939147cb941 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T121955220639Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66889701376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157809437696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66889701376, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66889701376, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_peak": 10756893696, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 172415254528, + "reserved_bytes_peak": 172415254528, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:55.220258Z", + "world_size": 8 + }, + "allocated_bytes_current": 66889701376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105834514944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889701376, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66889701376, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_peak": 10756893696, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 172415254528, + "reserved_bytes_peak": 172415254528, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:54.669299Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10756893696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 172415254528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172415254528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 894, + "segment_count_current_delta": 0, + "segment_count_peak": 894, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T121955220639Z.pickle", + "timestamp_utc": "2026-08-01T12:19:55.226692Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T121955220639Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T121955220639Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b8357283ee0d81390a7fcbe12190cbf9a9318653 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T121955220639Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:868b56b3ba57a582f9091aad433d134045db0589f22ce7ae082a84b70ce37713 +size 267698 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T122049955625Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T122049955625Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0e81a8d7c5f962b3cc8cc514779b5ab3295f6fd5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T122049955625Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66856067072, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 157809437696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66856067072, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66856067072, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 6959489024, + "inactive_split_bytes_peak": 10756893696, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 174378188800, + "reserved_bytes_peak": 174378188800, + "segment_count_current": 903, + "segment_count_peak": 903, + "timestamp_utc": "2026-08-01T12:20:49.955142Z", + "world_size": 8 + }, + "allocated_bytes_current": 66856067072, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105834514944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66889701376, + "active_bytes_peak": 157809437696, + "allocated_bytes_current": 66889701376, + "allocated_bytes_peak": 105834514944, + "device": 4, + "inactive_split_bytes_current": 7213164544, + "inactive_split_bytes_peak": 10756893696, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 172415254528, + "reserved_bytes_peak": 172415254528, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:55.227612Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6959489024, + "inactive_split_bytes_current_delta": -253675520, + "inactive_split_bytes_peak": 10756893696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 174378188800, + "reserved_bytes_current_delta": 1962934272, + "reserved_bytes_peak": 174378188800, + "reserved_bytes_peak_delta": 1962934272, + "segment_count_current": 903, + "segment_count_current_delta": 9, + "segment_count_peak": 903, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T122049955625Z.pickle", + "timestamp_utc": "2026-08-01T12:20:50.047850Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T122049955625Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T122049955625Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5fa9f166e4660484f13d99096b957bae7bf504c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T122049955625Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62da75cbecf971d1814fbd2ea6bad5c1e04fd97019f4da1a096eb178b4e67cba +size 3939199 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T121947731982Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T121947731982Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0b365f3f39aa7bc6a89cebb04235c90a92aa6d1f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T121947731982Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 31639604224, + "active_bytes_peak": 82189568000, + "active_bytes_peak_delta": 46723231744, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 82189568000, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 82189568000, + "device": 5, + "inactive_split_bytes_current": 7223417856, + "inactive_split_bytes_peak": 7223418880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 99107209216, + "reserved_bytes_peak": 99107209216, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T12:19:47.731373Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 31639604224, + "allocated_bytes_peak": 82189568000, + "allocated_bytes_peak_delta": 46723231744, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:18:38.035685Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7223417856, + "inactive_split_bytes_current_delta": 7090598912, + "inactive_split_bytes_peak": 7223418880, + "inactive_split_bytes_peak_delta": 7063032832, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 99107209216, + "reserved_bytes_current_delta": 59863203840, + "reserved_bytes_peak": 99107209216, + "reserved_bytes_peak_delta": 59863203840, + "segment_count_current": 505, + "segment_count_current_delta": 434, + "segment_count_peak": 505, + "segment_count_peak_delta": 434, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T121947731982Z.pickle", + "timestamp_utc": "2026-08-01T12:19:47.840620Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T121947731982Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T121947731982Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8618819ede61c2a3371c895e54f22147209d9809 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T121947731982Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:359c9c212ed0a4104c6a1d9a2485e6e7363bc09f15b3c80dca3f9c3e551a698c +size 5384607 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T121948700804Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T121948700804Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6bf8ccc490a2f216400928c28a2461f0d4c77099 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T121948700804Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105828004352, + "active_bytes_peak_delta": 23638436352, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828004352, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828004352, + "device": 5, + "inactive_split_bytes_current": 7236000768, + "inactive_split_bytes_peak": 7766498304, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108680708096, + "reserved_bytes_peak": 108680708096, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T12:19:48.700390Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105828004352, + "allocated_bytes_peak_delta": 23638436352, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 82189568000, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 82189568000, + "device": 5, + "inactive_split_bytes_current": 7223417856, + "inactive_split_bytes_peak": 7223418880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 99107209216, + "reserved_bytes_peak": 99107209216, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T12:19:47.859891Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7236000768, + "inactive_split_bytes_current_delta": 12582912, + "inactive_split_bytes_peak": 7766498304, + "inactive_split_bytes_peak_delta": 543079424, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108680708096, + "reserved_bytes_current_delta": 9573498880, + "reserved_bytes_peak": 108680708096, + "reserved_bytes_peak_delta": 9573498880, + "segment_count_current": 538, + "segment_count_current_delta": 33, + "segment_count_peak": 538, + "segment_count_peak_delta": 33, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T121948700804Z.pickle", + "timestamp_utc": "2026-08-01T12:19:48.804437Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T121948700804Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T121948700804Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a80da8762421c353d637c3b22706cadcdd99354e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T121948700804Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dbab9ed3e54c6ee3dba5eac511f67b4c3ea5094e80f78b4a0d2706d7eb4389c +size 5584881 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T121949663012Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T121949663012Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c2ec93dfa13490eba0822df3377cad7098d753d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T121949663012Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105828004352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828004352, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828004352, + "device": 5, + "inactive_split_bytes_current": 7240195072, + "inactive_split_bytes_peak": 8179719168, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 110033371136, + "reserved_bytes_peak": 110033371136, + "segment_count_current": 542, + "segment_count_peak": 542, + "timestamp_utc": "2026-08-01T12:19:49.662612Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105828004352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828004352, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828004352, + "device": 5, + "inactive_split_bytes_current": 7236000768, + "inactive_split_bytes_peak": 7766498304, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108680708096, + "reserved_bytes_peak": 108680708096, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T12:19:48.819214Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7240195072, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8179719168, + "inactive_split_bytes_peak_delta": 413220864, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 110033371136, + "reserved_bytes_current_delta": 1352663040, + "reserved_bytes_peak": 110033371136, + "reserved_bytes_peak_delta": 1352663040, + "segment_count_current": 542, + "segment_count_current_delta": 4, + "segment_count_peak": 542, + "segment_count_peak_delta": 4, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T121949663012Z.pickle", + "timestamp_utc": "2026-08-01T12:19:49.766750Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T121949663012Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T121949663012Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e125b8b783e6f7144dece5b353018532829638d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T121949663012Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:309aae6d2d1fd976f692b40a3b1ee3eaf15c7c6a13ab9d5efd34d1b0dac77330 +size 5563989 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T121952281561Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T121952281561Z.json new file mode 100644 index 0000000000000000000000000000000000000000..265d2dbb75feb013e6126bdbc739a90c515fb514 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T121952281561Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 51976820224, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_peak": 11097828864, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 172318785536, + "reserved_bytes_peak": 172318785536, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:52.280977Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 3338752, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828004352, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828004352, + "device": 5, + "inactive_split_bytes_current": 7240195072, + "inactive_split_bytes_peak": 8179719168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 110033371136, + "reserved_bytes_peak": 110033371136, + "segment_count_current": 542, + "segment_count_peak": 542, + "timestamp_utc": "2026-08-01T12:19:49.781476Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_current_delta": -123998208, + "inactive_split_bytes_peak": 11097828864, + "inactive_split_bytes_peak_delta": 2918109696, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 172318785536, + "reserved_bytes_current_delta": 62285414400, + "reserved_bytes_peak": 172318785536, + "reserved_bytes_peak_delta": 62285414400, + "segment_count_current": 897, + "segment_count_current_delta": 355, + "segment_count_peak": 897, + "segment_count_peak_delta": 355, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T121952281561Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.400611Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T121952281561Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T121952281561Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..555114197866f2ffbf319cb491724fea550f8a36 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T121952281561Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db590c2d36b3afdbf117d9466b4a7a0bea3f5e8a69213b6bfca3588587e856f9 +size 6583781 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T121952972702Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T121952972702Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2191bbf5a493bdb5629aa5ead7a4d75d36771505 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T121952972702Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_peak": 11097828864, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 172318785536, + "reserved_bytes_peak": 172318785536, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:52.972247Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_peak": 11097828864, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 172318785536, + "reserved_bytes_peak": 172318785536, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:52.417186Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11097828864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 172318785536, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172318785536, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 897, + "segment_count_current_delta": 0, + "segment_count_peak": 897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T121952972702Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.984118Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T121952972702Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T121952972702Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cfcfd0ff2168ed16385993d6a1a1d57ae5a85c6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T121952972702Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcc750bb29814bf88d32068b26532086fc11a2ef22be2f4a95cfff85ff428e3b +size 261759 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T121953540725Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T121953540725Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c9254dfa06d9416f3f3df3e6319f6f67b9148e0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T121953540725Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_peak": 11097828864, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 172318785536, + "reserved_bytes_peak": 172318785536, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:53.540326Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_peak": 11097828864, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 172318785536, + "reserved_bytes_peak": 172318785536, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:52.985173Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11097828864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 172318785536, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172318785536, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 897, + "segment_count_current_delta": 0, + "segment_count_peak": 897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T121953540725Z.pickle", + "timestamp_utc": "2026-08-01T12:19:53.547289Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T121953540725Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T121953540725Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e4d87b98c8a0796d352159ef702471bde66f730 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T121953540725Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6d75f16c6fa6c36720c81537c3a12ba97cfdc6e2e05b5fa10e5da36c7437687 +size 261759 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T121954101908Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T121954101908Z.json new file mode 100644 index 0000000000000000000000000000000000000000..44701ce1b1e8f810be346ba23123e67852cb5852 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T121954101908Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_peak": 11097828864, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 172318785536, + "reserved_bytes_peak": 172318785536, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:54.101519Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_peak": 11097828864, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 172318785536, + "reserved_bytes_peak": 172318785536, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:53.548230Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11097828864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 172318785536, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172318785536, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 897, + "segment_count_current_delta": 0, + "segment_count_peak": 897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T121954101908Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.107851Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T121954101908Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T121954101908Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c13eda8c97db850f8d8b0fc5feab04e5390209f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T121954101908Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b83ff27fee0c15bb0b9d94f1dfb2d1d76dfa855695e49ea5109de3aad0364ea +size 261759 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T121954662046Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T121954662046Z.json new file mode 100644 index 0000000000000000000000000000000000000000..47dbfe8da7f5cfd7a3d8019e06a4e257ace4d273 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T121954662046Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_peak": 11097828864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 172318785536, + "reserved_bytes_peak": 172318785536, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:54.661646Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_peak": 11097828864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 172318785536, + "reserved_bytes_peak": 172318785536, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:54.108797Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11097828864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 172318785536, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172318785536, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 897, + "segment_count_current_delta": 0, + "segment_count_peak": 897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T121954662046Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.667995Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T121954662046Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T121954662046Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..093f1f9e6c03c29d514be960aa070ae278ddc74d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T121954662046Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:989654d119773fb7b0eab3ca4dcc5326c04adb6671d012bafcdb3c6a16842ee3 +size 261759 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T121955220809Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T121955220809Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0587457176c9fe06d2a990ba88328c749f41269f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T121955220809Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_peak": 11097828864, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 172318785536, + "reserved_bytes_peak": 172318785536, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:55.220396Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_peak": 11097828864, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 172318785536, + "reserved_bytes_peak": 172318785536, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:54.668922Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11097828864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 172318785536, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172318785536, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 897, + "segment_count_current_delta": 0, + "segment_count_peak": 897, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T121955220809Z.pickle", + "timestamp_utc": "2026-08-01T12:19:55.228925Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T121955220809Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T121955220809Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b95851906fa7171f6683f77edb9faf174b05100 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T121955220809Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3d04700b6cf7a9d280a75af7686e82c16d1e93055f9210fdd4dd8d4dea4bbdc +size 261759 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T122049955616Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T122049955616Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1591346baecd22906b5aed61b052c62019ca4bf1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T122049955616Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854468608, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 157804824576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854468608, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66854468608, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7097402368, + "inactive_split_bytes_peak": 11097828864, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 174281719808, + "reserved_bytes_peak": 174281719808, + "segment_count_current": 906, + "segment_count_peak": 906, + "timestamp_utc": "2026-08-01T12:20:49.955127Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854468608, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105831343104, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157804824576, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105831343104, + "device": 5, + "inactive_split_bytes_current": 7116196864, + "inactive_split_bytes_peak": 11097828864, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 172318785536, + "reserved_bytes_peak": 172318785536, + "segment_count_current": 897, + "segment_count_peak": 897, + "timestamp_utc": "2026-08-01T12:19:55.229848Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7097402368, + "inactive_split_bytes_current_delta": -18794496, + "inactive_split_bytes_peak": 11097828864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 174281719808, + "reserved_bytes_current_delta": 1962934272, + "reserved_bytes_peak": 174281719808, + "reserved_bytes_peak_delta": 1962934272, + "segment_count_current": 906, + "segment_count_current_delta": 9, + "segment_count_peak": 906, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T122049955616Z.pickle", + "timestamp_utc": "2026-08-01T12:20:50.040942Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T122049955616Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T122049955616Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f24881afcbc13c8811274f47650310962381ad39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T122049955616Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:253d1467af4af660077140e994c86c963664ff64cc4ba24d4592e8e8bfeb3c0e +size 3932822 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T121947732152Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T121947732152Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f24e3e7c79f8786913982e4504ec360196661c59 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T121947732152Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 31639604224, + "active_bytes_peak": 82189568000, + "active_bytes_peak_delta": 46723231744, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 82189568000, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 82189568000, + "device": 6, + "inactive_split_bytes_current": 7223417856, + "inactive_split_bytes_peak": 7223418880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98637447168, + "reserved_bytes_peak": 98637447168, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:19:47.731534Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 31639604224, + "allocated_bytes_peak": 82189568000, + "allocated_bytes_peak_delta": 46723231744, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:18:38.035655Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7223417856, + "inactive_split_bytes_current_delta": 7090598912, + "inactive_split_bytes_peak": 7223418880, + "inactive_split_bytes_peak_delta": 7063032832, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98637447168, + "reserved_bytes_current_delta": 59393441792, + "reserved_bytes_peak": 98637447168, + "reserved_bytes_peak_delta": 59393441792, + "segment_count_current": 503, + "segment_count_current_delta": 432, + "segment_count_peak": 503, + "segment_count_peak_delta": 432, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T121947732152Z.pickle", + "timestamp_utc": "2026-08-01T12:19:47.846066Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T121947732152Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T121947732152Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5704db30e055c1f370d7a671a7ed0e4d9bc96a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T121947732152Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:212b822582b66ed4bbdcfd338186fbf90c757322c59d68b6cad4dddd6a7d0ca3 +size 5384243 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T121948700833Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T121948700833Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c9260ae433e7656a2c625eb160152166d57e24d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T121948700833Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105828004352, + "active_bytes_peak_delta": 23638436352, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828004352, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828004352, + "device": 6, + "inactive_split_bytes_current": 7470881792, + "inactive_split_bytes_peak": 8003394560, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108680708096, + "reserved_bytes_peak": 108680708096, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T12:19:48.700429Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105828004352, + "allocated_bytes_peak_delta": 23638436352, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 82189568000, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 82189568000, + "device": 6, + "inactive_split_bytes_current": 7223417856, + "inactive_split_bytes_peak": 7223418880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98637447168, + "reserved_bytes_peak": 98637447168, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:19:47.863489Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7470881792, + "inactive_split_bytes_current_delta": 247463936, + "inactive_split_bytes_peak": 8003394560, + "inactive_split_bytes_peak_delta": 779975680, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108680708096, + "reserved_bytes_current_delta": 10043260928, + "reserved_bytes_peak": 108680708096, + "reserved_bytes_peak_delta": 10043260928, + "segment_count_current": 538, + "segment_count_current_delta": 35, + "segment_count_peak": 538, + "segment_count_peak_delta": 35, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T121948700833Z.pickle", + "timestamp_utc": "2026-08-01T12:19:48.803695Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T121948700833Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T121948700833Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c201b3dcd820fee8f5080224360a809974ca38f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T121948700833Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cad33fb3366154c496fd1f3bfc829ffecb04a3715298854b4f7da446691acac0 +size 5584878 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T121949663047Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T121949663047Z.json new file mode 100644 index 0000000000000000000000000000000000000000..01a4954c43d820a45e7862c5fe540de9790cf80a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T121949663047Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105828004352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828004352, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828004352, + "device": 6, + "inactive_split_bytes_current": 7003216896, + "inactive_split_bytes_peak": 8132910080, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109150470144, + "reserved_bytes_peak": 109150470144, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T12:19:49.662645Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105828004352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828004352, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828004352, + "device": 6, + "inactive_split_bytes_current": 7470881792, + "inactive_split_bytes_peak": 8003394560, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108680708096, + "reserved_bytes_peak": 108680708096, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T12:19:48.816831Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7003216896, + "inactive_split_bytes_current_delta": -467664896, + "inactive_split_bytes_peak": 8132910080, + "inactive_split_bytes_peak_delta": 129515520, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109150470144, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 109150470144, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 540, + "segment_count_current_delta": 2, + "segment_count_peak": 540, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T121949663047Z.pickle", + "timestamp_utc": "2026-08-01T12:19:49.765740Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T121949663047Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T121949663047Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd9a3a9e4cbdecfaf108d59cdf71f0b4fe862ba5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T121949663047Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:753cb81cfc8c1321f8bcfffee25e6a1e35efbc1586f7aaec7e16c8ea5720303a +size 5577197 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T121952281560Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T121952281560Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a2dfa532d54c5daf1c22560fb9910c3c190bede --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T121952281560Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 51982980608, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_peak": 10622692864, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 172079710208, + "reserved_bytes_peak": 172079710208, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:52.281000Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 8582144, + "before": { + "active_bytes_current": 66871059456, + "active_bytes_peak": 105828004352, + "allocated_bytes_current": 66871059456, + "allocated_bytes_peak": 105828004352, + "device": 6, + "inactive_split_bytes_current": 7003216896, + "inactive_split_bytes_peak": 8132910080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 109150470144, + "reserved_bytes_peak": 109150470144, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T12:19:49.779314Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_current_delta": 115077120, + "inactive_split_bytes_peak": 10622692864, + "inactive_split_bytes_peak_delta": 2489782784, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 172079710208, + "reserved_bytes_current_delta": 62929240064, + "reserved_bytes_peak": 172079710208, + "reserved_bytes_peak_delta": 62929240064, + "segment_count_current": 894, + "segment_count_current_delta": 354, + "segment_count_peak": 894, + "segment_count_peak_delta": 354, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T121952281560Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.399075Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T121952281560Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T121952281560Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..159d9848fcc6bdaf41650840bcb7e4452a40dc6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T121952281560Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a704241ecaf0488610e49e2f744ec24adfb3cb61921d2e0ff3619063def575b +size 6583029 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T121952972713Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T121952972713Z.json new file mode 100644 index 0000000000000000000000000000000000000000..78c8ca42f770052e4444022d012f9bc77b2a51f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T121952972713Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_peak": 10622692864, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 172079710208, + "reserved_bytes_peak": 172079710208, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:52.972257Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_peak": 10622692864, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 172079710208, + "reserved_bytes_peak": 172079710208, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:52.413750Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10622692864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 172079710208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172079710208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 894, + "segment_count_current_delta": 0, + "segment_count_peak": 894, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T121952972713Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.979366Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T121952972713Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T121952972713Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0143efb963fc2d3af7f17c47a3d972d47d6b43b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T121952972713Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05cf9fb1e53c2649c41fa7e4d776ac93f5bcd76d18d53b1f40645eabdde9b1c6 +size 260286 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T121953540718Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T121953540718Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e1300b028303b986b511e5113040488b9a7c387b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T121953540718Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_peak": 10622692864, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 172079710208, + "reserved_bytes_peak": 172079710208, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:53.540317Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_peak": 10622692864, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 172079710208, + "reserved_bytes_peak": 172079710208, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:52.980318Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10622692864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 172079710208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172079710208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 894, + "segment_count_current_delta": 0, + "segment_count_peak": 894, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T121953540718Z.pickle", + "timestamp_utc": "2026-08-01T12:19:53.546909Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T121953540718Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T121953540718Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca2c02dbc54da054cd7a8432b92fec32a6c0f8a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T121953540718Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59ec63dae4c93f8dd4b2643819d959991f9b2ea3c0e7c6afb140355f3e9011cd +size 260286 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T121954101946Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T121954101946Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e2ea3b2e5bd9523bd4220352e999206134a1986 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T121954101946Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_peak": 10622692864, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 172079710208, + "reserved_bytes_peak": 172079710208, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:54.101540Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_peak": 10622692864, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 172079710208, + "reserved_bytes_peak": 172079710208, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:53.547848Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10622692864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 172079710208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172079710208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 894, + "segment_count_current_delta": 0, + "segment_count_peak": 894, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T121954101946Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.107906Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T121954101946Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T121954101946Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3830b5866767d5712259c0b26067af9176398fef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T121954101946Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7105a9054e40c3f4aa8bf81e15c763268d73281edaf14a633353d1eb1054290 +size 260286 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T121954662088Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T121954662088Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f12b92d86e9e925f0b23a39328ed738c2621e3d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T121954662088Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_peak": 10622692864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 172079710208, + "reserved_bytes_peak": 172079710208, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:54.661685Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_peak": 10622692864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 172079710208, + "reserved_bytes_peak": 172079710208, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:54.108832Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10622692864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 172079710208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172079710208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 894, + "segment_count_current_delta": 0, + "segment_count_peak": 894, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T121954662088Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.668067Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T121954662088Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T121954662088Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..306ffb88392e0e4fd1597e8c793584d1aab09f52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T121954662088Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7cb488ad9da472cd3b9cab137c758eb91f25329354e28e3b3e714b2842b233f +size 260286 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T121955220760Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T121955220760Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e5837359ff2d15751dacc23a3cb03bd0ee45ce0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T121955220760Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888102912, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_peak": 10622692864, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 172079710208, + "reserved_bytes_peak": 172079710208, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:55.220322Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888102912, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_peak": 10622692864, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 172079710208, + "reserved_bytes_peak": 172079710208, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:54.668995Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10622692864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 172079710208, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 172079710208, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 894, + "segment_count_current_delta": 0, + "segment_count_peak": 894, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T121955220760Z.pickle", + "timestamp_utc": "2026-08-01T12:19:55.227098Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T121955220760Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T121955220760Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..35681e355dea7a4344ef3c3b64cbab39901616f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T121955220760Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf10dc7c11d0a3e0141c42f50d0996280877eb86a53db4e8c3a6eff502c00c71 +size 260286 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T122049955612Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T122049955612Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aa40c55c7b35e475b862365f0778d6d3e5bd6ed8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T122049955612Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854468608, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 157810984960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854468608, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66854468608, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7097402368, + "inactive_split_bytes_peak": 10622692864, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 174277525504, + "reserved_bytes_peak": 174277525504, + "segment_count_current": 904, + "segment_count_peak": 904, + "timestamp_utc": "2026-08-01T12:20:49.955132Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854468608, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105836586496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888102912, + "active_bytes_peak": 157810984960, + "allocated_bytes_current": 66888102912, + "allocated_bytes_peak": 105836586496, + "device": 6, + "inactive_split_bytes_current": 7118294016, + "inactive_split_bytes_peak": 10622692864, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 172079710208, + "reserved_bytes_peak": 172079710208, + "segment_count_current": 894, + "segment_count_peak": 894, + "timestamp_utc": "2026-08-01T12:19:55.228031Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7097402368, + "inactive_split_bytes_current_delta": -20891648, + "inactive_split_bytes_peak": 10622692864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 174277525504, + "reserved_bytes_current_delta": 2197815296, + "reserved_bytes_peak": 174277525504, + "reserved_bytes_peak_delta": 2197815296, + "segment_count_current": 904, + "segment_count_current_delta": 10, + "segment_count_peak": 904, + "segment_count_peak_delta": 10, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T122049955612Z.pickle", + "timestamp_utc": "2026-08-01T12:20:50.040165Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T122049955612Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T122049955612Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..036b473ff018a69db5748e55a12fbcc104414f4e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T122049955612Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2223d80cfde17f2e58fbb7184d91650444c65e4b0ac35d03843bbfafb3952d1b +size 3931451 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T121947731991Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T121947731991Z.json new file mode 100644 index 0000000000000000000000000000000000000000..685baa4aa8d97b7de2f1a7825bf9b308a59fc2f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T121947731991Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059968, + "active_bytes_current_delta": 31639604736, + "active_bytes_peak": 82188126208, + "active_bytes_peak_delta": 46721789952, + "after": { + "active_bytes_current": 66871059968, + "active_bytes_peak": 82188126208, + "allocated_bytes_current": 66871059968, + "allocated_bytes_peak": 82188126208, + "device": 7, + "inactive_split_bytes_current": 7045159424, + "inactive_split_bytes_peak": 7045160960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 98459189248, + "reserved_bytes_peak": 98459189248, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:19:47.731470Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059968, + "allocated_bytes_current_delta": 31639604736, + "allocated_bytes_peak": 82188126208, + "allocated_bytes_peak_delta": 46721789952, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:18:38.035695Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7045159424, + "inactive_split_bytes_current_delta": 6912340480, + "inactive_split_bytes_peak": 7045160960, + "inactive_split_bytes_peak_delta": 6884774912, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 98459189248, + "reserved_bytes_current_delta": 59215183872, + "reserved_bytes_peak": 98459189248, + "reserved_bytes_peak_delta": 59215183872, + "segment_count_current": 503, + "segment_count_current_delta": 432, + "segment_count_peak": 503, + "segment_count_peak_delta": 432, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T121947731991Z.pickle", + "timestamp_utc": "2026-08-01T12:19:47.843519Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T121947731991Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T121947731991Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f885a66037099f2cd31f7d1c45e44db8743fb8d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T121947731991Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd7d79a5df891a0711adfd1eaef5120b12e9eb445fd4483ef3bb24c57d0d8ac5 +size 5385811 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T121948700984Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T121948700984Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b32b596c89e06e396e55d229e770b992f629beea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T121948700984Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105826046976, + "active_bytes_peak_delta": 23637920768, + "after": { + "active_bytes_current": 66871059968, + "active_bytes_peak": 105826046976, + "allocated_bytes_current": 66871059968, + "allocated_bytes_peak": 105826046976, + "device": 7, + "inactive_split_bytes_current": 7059839488, + "inactive_split_bytes_peak": 7590418944, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108208848896, + "reserved_bytes_peak": 108208848896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T12:19:48.700502Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105826046976, + "allocated_bytes_peak_delta": 23637920768, + "before": { + "active_bytes_current": 66871059968, + "active_bytes_peak": 82188126208, + "allocated_bytes_current": 66871059968, + "allocated_bytes_peak": 82188126208, + "device": 7, + "inactive_split_bytes_current": 7045159424, + "inactive_split_bytes_peak": 7045160960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 98459189248, + "reserved_bytes_peak": 98459189248, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:19:47.863267Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7059839488, + "inactive_split_bytes_current_delta": 14680064, + "inactive_split_bytes_peak": 7590418944, + "inactive_split_bytes_peak_delta": 545257984, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108208848896, + "reserved_bytes_current_delta": 9749659648, + "reserved_bytes_peak": 108208848896, + "reserved_bytes_peak_delta": 9749659648, + "segment_count_current": 538, + "segment_count_current_delta": 35, + "segment_count_peak": 538, + "segment_count_peak_delta": 35, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T121948700984Z.pickle", + "timestamp_utc": "2026-08-01T12:19:48.807621Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T121948700984Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T121948700984Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..759b4d98196d740e049b43faeb05ba1cdaaa94d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T121948700984Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67cbf686f35c828ef6cec73ec2f6d53ba9a7034bfd21034d2694d1e8b158936b +size 5586153 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T121949663138Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T121949663138Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3d832110e5b83bce393e0c11b71d62803db37309 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T121949663138Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66871059968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105826046976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66871059968, + "active_bytes_peak": 105826046976, + "allocated_bytes_current": 66871059968, + "allocated_bytes_peak": 105826046976, + "device": 7, + "inactive_split_bytes_current": 6827055616, + "inactive_split_bytes_peak": 7719721472, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108678610944, + "reserved_bytes_peak": 108678610944, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T12:19:49.662654Z", + "world_size": 8 + }, + "allocated_bytes_current": 66871059968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105826046976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66871059968, + "active_bytes_peak": 105826046976, + "allocated_bytes_current": 66871059968, + "allocated_bytes_peak": 105826046976, + "device": 7, + "inactive_split_bytes_current": 7059839488, + "inactive_split_bytes_peak": 7590418944, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108208848896, + "reserved_bytes_peak": 108208848896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T12:19:48.823195Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6827055616, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 7719721472, + "inactive_split_bytes_peak_delta": 129302528, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108678610944, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 108678610944, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 540, + "segment_count_current_delta": 2, + "segment_count_peak": 540, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T121949663138Z.pickle", + "timestamp_utc": "2026-08-01T12:19:49.767819Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T121949663138Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T121949663138Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03e03d87e91e25eb3d3890fe6eab89ef5a310fc8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T121949663138Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44bdd3647d7c249d6369506c3542386cbb6d9fe5edc6a94a9ac86f76b63404a3 +size 5580016 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T121952281644Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T121952281644Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94c86178fa696f64fd4601fe466b04e39db6db11 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T121952281644Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888103424, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 157804825088, + "active_bytes_peak_delta": 51978778112, + "after": { + "active_bytes_current": 66888103424, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66888103424, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_peak": 10686786560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:52.281060Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888103424, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105831343616, + "allocated_bytes_peak_delta": 5296640, + "before": { + "active_bytes_current": 66871059968, + "active_bytes_peak": 105826046976, + "allocated_bytes_current": 66871059968, + "allocated_bytes_peak": 105826046976, + "device": 7, + "inactive_split_bytes_current": 6827055616, + "inactive_split_bytes_peak": 7719721472, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108678610944, + "reserved_bytes_peak": 108678610944, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T12:19:49.782618Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_current_delta": 112979968, + "inactive_split_bytes_peak": 10686786560, + "inactive_split_bytes_peak_delta": 2967065088, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 171903549440, + "reserved_bytes_current_delta": 63224938496, + "reserved_bytes_peak": 171903549440, + "reserved_bytes_peak_delta": 63224938496, + "segment_count_current": 895, + "segment_count_current_delta": 355, + "segment_count_peak": 895, + "segment_count_peak_delta": 355, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T121952281644Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.401807Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T121952281644Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T121952281644Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..35a9b0e772b2c3360a69b1948e71fd14c83df08c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T121952281644Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ab212aec92bd531e8e2ecdb20eda7ddcd855d44f675cb4f672d8dc5858f79ed +size 6585838 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T121952972849Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T121952972849Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bf21e8b7246c85fc4b2998f909fd3a9aaf5dad8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T121952972849Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888103424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804825088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888103424, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66888103424, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_peak": 10686786560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:52.972366Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888103424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888103424, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66888103424, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_peak": 10686786560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:52.418075Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10686786560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 171903549440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171903549440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 895, + "segment_count_current_delta": 0, + "segment_count_peak": 895, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T121952972849Z.pickle", + "timestamp_utc": "2026-08-01T12:19:52.980439Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T121952972849Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T121952972849Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6974b39eb403f0aed40ec26fb91b87b10519f792 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T121952972849Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37ba7ef9a5497add538edfa1d04224f0038e8e93171987539b05348f66e348c9 +size 263094 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T121953540740Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T121953540740Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0aa2bb3a93843b970be22850a42bd51688217418 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T121953540740Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888103424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804825088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888103424, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66888103424, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_peak": 10686786560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:53.540345Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888103424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888103424, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66888103424, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_peak": 10686786560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:52.981414Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10686786560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 171903549440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171903549440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 895, + "segment_count_current_delta": 0, + "segment_count_peak": 895, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T121953540740Z.pickle", + "timestamp_utc": "2026-08-01T12:19:53.547111Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T121953540740Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T121953540740Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5830605fd664ac538f512d5592fd0539b0255d5a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T121953540740Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15167c4f0a54cae85973bea36761b1420b3f69f4feeccce5a2ea9416b146809f +size 263094 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T121954101957Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T121954101957Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae9626b9487f7a4d5736bbce223b44f0738dae8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T121954101957Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888103424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804825088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888103424, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66888103424, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_peak": 10686786560, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:54.101564Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888103424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888103424, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66888103424, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_peak": 10686786560, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:53.548053Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10686786560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 171903549440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171903549440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 895, + "segment_count_current_delta": 0, + "segment_count_peak": 895, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T121954101957Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.107931Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T121954101957Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T121954101957Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1509f1a678327f586ceba49f63d1862dd2953369 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T121954101957Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffc001afd2660e1a5ccc9bd4cb4612102561f0ee1a4c8656f0253b9090c87f99 +size 263094 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T121954662081Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T121954662081Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15ac7199879aef3d2dc00f325e34a7845e68b81f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T121954662081Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888103424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804825088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888103424, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66888103424, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_peak": 10686786560, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:54.661686Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888103424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888103424, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66888103424, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_peak": 10686786560, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:54.108883Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10686786560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 171903549440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171903549440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 895, + "segment_count_current_delta": 0, + "segment_count_peak": 895, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T121954662081Z.pickle", + "timestamp_utc": "2026-08-01T12:19:54.668134Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T121954662081Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T121954662081Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6544b6ff556afab33a694b220b7cd815cf41335 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T121954662081Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff8743778e61b7de558c3eea6017d766f92c6ddc637fbafefec6c12a46213cf1 +size 263094 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T121955220671Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T121955220671Z.json new file mode 100644 index 0000000000000000000000000000000000000000..83a052c826090a9be515849c88ac9c32a1fbbcd0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T121955220671Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66888103424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 157804825088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66888103424, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66888103424, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_peak": 10686786560, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:55.220284Z", + "world_size": 8 + }, + "allocated_bytes_current": 66888103424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105831343616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888103424, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66888103424, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_peak": 10686786560, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:54.669078Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10686786560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 171903549440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 171903549440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 895, + "segment_count_current_delta": 0, + "segment_count_peak": 895, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T121955220671Z.pickle", + "timestamp_utc": "2026-08-01T12:19:55.226568Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T121955220671Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T121955220671Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..45feba47950414885c347ce6d8e9d3f01aa9d4ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T121955220671Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:179b0f1dc4122611b2a666fad38ac256901559f6f0abfab6890af5e7afca543d +size 263094 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T122049955626Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T122049955626Z.json new file mode 100644 index 0000000000000000000000000000000000000000..09fd8e40a82ab36accce243a0dc7631fcb3997e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T122049955626Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854469120, + "active_bytes_current_delta": -33634304, + "active_bytes_peak": 157804825088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854469120, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66854469120, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6684262912, + "inactive_split_bytes_peak": 10686786560, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 173631602688, + "reserved_bytes_peak": 173631602688, + "segment_count_current": 903, + "segment_count_peak": 903, + "timestamp_utc": "2026-08-01T12:20:49.955159Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854469120, + "allocated_bytes_current_delta": -33634304, + "allocated_bytes_peak": 105831343616, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66888103424, + "active_bytes_peak": 157804825088, + "allocated_bytes_current": 66888103424, + "allocated_bytes_peak": 105831343616, + "device": 7, + "inactive_split_bytes_current": 6940035584, + "inactive_split_bytes_peak": 10686786560, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 171903549440, + "reserved_bytes_peak": 171903549440, + "segment_count_current": 895, + "segment_count_peak": 895, + "timestamp_utc": "2026-08-01T12:19:55.227488Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6684262912, + "inactive_split_bytes_current_delta": -255772672, + "inactive_split_bytes_peak": 10686786560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 173631602688, + "reserved_bytes_current_delta": 1728053248, + "reserved_bytes_peak": 173631602688, + "reserved_bytes_peak_delta": 1728053248, + "segment_count_current": 903, + "segment_count_current_delta": 8, + "segment_count_peak": 903, + "segment_count_peak_delta": 8, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T122049955626Z.pickle", + "timestamp_utc": "2026-08-01T12:20:50.040761Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T122049955626Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T122049955626Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..418ac9aad88760445b6de37ad24fa65a6009c354 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T122049955626Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1564e1394c8eb597bb54a8af902f82ea6a51ed867c993459b52a7b8d15fa591d +size 3933574 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T122346775856Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T122346775856Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e753cf2286ea6be749cc43dc85bf4500eca1b2e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T122346775856Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457978880, + "active_bytes_current_delta": 37226523648, + "active_bytes_peak": 82506579968, + "active_bytes_peak_delta": 47040243712, + "after": { + "active_bytes_current": 72457978880, + "active_bytes_peak": 82506579968, + "allocated_bytes_current": 72457978880, + "allocated_bytes_peak": 82161820160, + "device": 0, + "inactive_split_bytes_current": 7147813888, + "inactive_split_bytes_peak": 8344854528, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 101992890368, + "reserved_bytes_peak": 101992890368, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T12:23:46.775215Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457978880, + "allocated_bytes_current_delta": 37226523648, + "allocated_bytes_peak": 82161820160, + "allocated_bytes_peak_delta": 46695483904, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:22:38.711618Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7147813888, + "inactive_split_bytes_current_delta": 7012897792, + "inactive_split_bytes_peak": 8344854528, + "inactive_split_bytes_peak_delta": 8184222720, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 101992890368, + "reserved_bytes_current_delta": 62746787840, + "reserved_bytes_peak": 101992890368, + "reserved_bytes_peak_delta": 62746787840, + "segment_count_current": 476, + "segment_count_current_delta": 404, + "segment_count_peak": 476, + "segment_count_peak_delta": 404, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T122346775856Z.pickle", + "timestamp_utc": "2026-08-01T12:23:46.894863Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T122346775856Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T122346775856Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c73c942c735eb6c365aeb461bcad4e07e2352e58 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T122346775856Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a382cc99f7880fe0f0caaf6fe02707fcd0bdb08b0738933636287cf47ad4c799 +size 5952857 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T122347835853Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T122347835853Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b7828aecc904886d73a4b6b8e87766e53d46bbac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T122347835853Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457978880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106075088384, + "active_bytes_peak_delta": 23568508416, + "after": { + "active_bytes_current": 72457978880, + "active_bytes_peak": 106075088384, + "allocated_bytes_current": 72457978880, + "allocated_bytes_peak": 105796559360, + "device": 0, + "inactive_split_bytes_current": 8211069952, + "inactive_split_bytes_peak": 8344854528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 111222456320, + "reserved_bytes_peak": 111222456320, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T12:23:47.835437Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457978880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105796559360, + "allocated_bytes_peak_delta": 23634739200, + "before": { + "active_bytes_current": 72457978880, + "active_bytes_peak": 82506579968, + "allocated_bytes_current": 72457978880, + "allocated_bytes_peak": 82161820160, + "device": 0, + "inactive_split_bytes_current": 7147813888, + "inactive_split_bytes_peak": 8344854528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 101992890368, + "reserved_bytes_peak": 101992890368, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T12:23:46.916477Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8211069952, + "inactive_split_bytes_current_delta": 1063256064, + "inactive_split_bytes_peak": 8344854528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 111222456320, + "reserved_bytes_current_delta": 9229565952, + "reserved_bytes_peak": 111222456320, + "reserved_bytes_peak_delta": 9229565952, + "segment_count_current": 502, + "segment_count_current_delta": 26, + "segment_count_peak": 502, + "segment_count_peak_delta": 26, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T122347835853Z.pickle", + "timestamp_utc": "2026-08-01T12:23:47.945450Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T122347835853Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T122347835853Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4f770ded9ba738ab6183b60f3e1c180cae21fc33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T122347835853Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:589445e3ff0d48e77b1aa3b1c830887b3456cde22c7e2f3a46a79aac8d48128f +size 6087846 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T122348917245Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T122348917245Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f495b96a158bcafff613bcfbc26b54613284687a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T122348917245Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457978880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106077317120, + "active_bytes_peak_delta": 2228736, + "after": { + "active_bytes_current": 72457978880, + "active_bytes_peak": 106077317120, + "allocated_bytes_current": 72457978880, + "allocated_bytes_peak": 105799017984, + "device": 0, + "inactive_split_bytes_current": 7160396800, + "inactive_split_bytes_peak": 8344854528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 111222456320, + "reserved_bytes_peak": 111222456320, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T12:23:48.916835Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457978880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105799017984, + "allocated_bytes_peak_delta": 2458624, + "before": { + "active_bytes_current": 72457978880, + "active_bytes_peak": 106075088384, + "allocated_bytes_current": 72457978880, + "allocated_bytes_peak": 105796559360, + "device": 0, + "inactive_split_bytes_current": 8211069952, + "inactive_split_bytes_peak": 8344854528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 111222456320, + "reserved_bytes_peak": 111222456320, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T12:23:47.961813Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7160396800, + "inactive_split_bytes_current_delta": -1050673152, + "inactive_split_bytes_peak": 8344854528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 111222456320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111222456320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 502, + "segment_count_current_delta": 0, + "segment_count_peak": 502, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T122348917245Z.pickle", + "timestamp_utc": "2026-08-01T12:23:49.027358Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T122348917245Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T122348917245Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..17064250cbf86e14cc3a6d06ab53c1f82ccac5d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T122348917245Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29aaf5366f41e09187d750bf2d9f721a4cdb66488020249ef4f05441b14b2166 +size 6589098 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T122346775703Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T122346775703Z.json new file mode 100644 index 0000000000000000000000000000000000000000..af9a0e6ff07efa3b46f87566c718974f433d6830 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T122346775703Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 37226105856, + "active_bytes_peak": 82437071872, + "active_bytes_peak_delta": 46970735616, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 82437071872, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 82158526464, + "device": 1, + "inactive_split_bytes_current": 7095802880, + "inactive_split_bytes_peak": 7328372224, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 100795416576, + "reserved_bytes_peak": 100795416576, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:23:46.775083Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 37226105856, + "allocated_bytes_peak": 82158526464, + "allocated_bytes_peak_delta": 46692190208, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:22:38.711569Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7095802880, + "inactive_split_bytes_current_delta": 6962983936, + "inactive_split_bytes_peak": 7328372224, + "inactive_split_bytes_peak_delta": 7167986176, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 100795416576, + "reserved_bytes_current_delta": 61551411200, + "reserved_bytes_peak": 100795416576, + "reserved_bytes_peak_delta": 61551411200, + "segment_count_current": 467, + "segment_count_current_delta": 396, + "segment_count_peak": 467, + "segment_count_peak_delta": 396, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T122346775703Z.pickle", + "timestamp_utc": "2026-08-01T12:23:46.892185Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T122346775703Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T122346775703Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a16703375b8c6c4b3b155e002f46829a9f06d750 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T122346775703Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cb3876deb6843492fc196a8163d666c850f0198bee080fedd68d0641b89104e +size 5942584 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T122347835855Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T122347835855Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7559797217e2fb5345a69e4b09cca4a59d433cc1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T122347835855Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106077600256, + "active_bytes_peak_delta": 23640528384, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106077600256, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105799220736, + "device": 1, + "inactive_split_bytes_current": 8096144384, + "inactive_split_bytes_peak": 8889105408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 111547514880, + "reserved_bytes_peak": 111547514880, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T12:23:47.835449Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105799220736, + "allocated_bytes_peak_delta": 23640694272, + "before": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 82437071872, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 82158526464, + "device": 1, + "inactive_split_bytes_current": 7095802880, + "inactive_split_bytes_peak": 7328372224, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 100795416576, + "reserved_bytes_peak": 100795416576, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:23:46.911361Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8096144384, + "inactive_split_bytes_current_delta": 1000341504, + "inactive_split_bytes_peak": 8889105408, + "inactive_split_bytes_peak_delta": 1560733184, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 111547514880, + "reserved_bytes_current_delta": 10752098304, + "reserved_bytes_peak": 111547514880, + "reserved_bytes_peak_delta": 10752098304, + "segment_count_current": 495, + "segment_count_current_delta": 28, + "segment_count_peak": 495, + "segment_count_peak_delta": 28, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T122347835855Z.pickle", + "timestamp_utc": "2026-08-01T12:23:47.942047Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T122347835855Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T122347835855Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d2e3090cc37a083182dc35dca5537065fe1dd2c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T122347835855Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f5630be89a0a1dfc9bd7d8eef34bc741ce5d4a5c99fb8775f6e2f80ebe9e9cd +size 6067755 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T122348917274Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T122348917274Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b50cc96cd4a26ca75090b5304f3839bc5283595d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T122348917274Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106078724608, + "active_bytes_peak_delta": 1124352, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106078724608, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105801227776, + "device": 1, + "inactive_split_bytes_current": 8152767488, + "inactive_split_bytes_peak": 8889105408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 111547514880, + "reserved_bytes_peak": 111547514880, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T12:23:48.916861Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105801227776, + "allocated_bytes_peak_delta": 2007040, + "before": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106077600256, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105799220736, + "device": 1, + "inactive_split_bytes_current": 8096144384, + "inactive_split_bytes_peak": 8889105408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 111547514880, + "reserved_bytes_peak": 111547514880, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T12:23:47.956515Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8152767488, + "inactive_split_bytes_current_delta": 56623104, + "inactive_split_bytes_peak": 8889105408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 111547514880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111547514880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 495, + "segment_count_current_delta": 0, + "segment_count_peak": 495, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T122348917274Z.pickle", + "timestamp_utc": "2026-08-01T12:23:49.026281Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T122348917274Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T122348917274Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f6b947bf32e13231e2c5b6967a8ecd34752d85c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T122348917274Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f207d9f73dc6331a58a43e5292c22294f3646eb8079b30a553e446db0bf20800 +size 6588262 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T122346775691Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T122346775691Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b549515febc4961deb2248a5177a73c7a96631c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T122346775691Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 37226105856, + "active_bytes_peak": 82504004096, + "active_bytes_peak_delta": 47037667840, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 82504004096, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 82158526464, + "device": 2, + "inactive_split_bytes_current": 7095802880, + "inactive_split_bytes_peak": 7553459200, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 100795416576, + "reserved_bytes_peak": 100795416576, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:23:46.775089Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 37226105856, + "allocated_bytes_peak": 82158526464, + "allocated_bytes_peak_delta": 46692190208, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:22:38.711745Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7095802880, + "inactive_split_bytes_current_delta": 6962983936, + "inactive_split_bytes_peak": 7553459200, + "inactive_split_bytes_peak_delta": 7393073152, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 100795416576, + "reserved_bytes_current_delta": 61551411200, + "reserved_bytes_peak": 100795416576, + "reserved_bytes_peak_delta": 61551411200, + "segment_count_current": 467, + "segment_count_current_delta": 396, + "segment_count_peak": 467, + "segment_count_peak_delta": 396, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T122346775691Z.pickle", + "timestamp_utc": "2026-08-01T12:23:46.890728Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T122346775691Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T122346775691Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b949b21f69b1d31b608208401903fc249f406e72 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T122346775691Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65345be048e38effe3aeb95df5a135b9974f1e563718436c03b8696ea7ef20ac +size 5942610 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T122347835939Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T122347835939Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2eb05ff1d647e92efc278f4d0d294b1a4d0dcda7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T122347835939Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106077354496, + "active_bytes_peak_delta": 23573350400, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106077354496, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105799068672, + "device": 2, + "inactive_split_bytes_current": 7802543104, + "inactive_split_bytes_peak": 9144410112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 111432171520, + "reserved_bytes_peak": 111432171520, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T12:23:47.835463Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105799068672, + "allocated_bytes_peak_delta": 23640542208, + "before": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 82504004096, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 82158526464, + "device": 2, + "inactive_split_bytes_current": 7095802880, + "inactive_split_bytes_peak": 7553459200, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 100795416576, + "reserved_bytes_peak": 100795416576, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:23:46.907552Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7802543104, + "inactive_split_bytes_current_delta": 706740224, + "inactive_split_bytes_peak": 9144410112, + "inactive_split_bytes_peak_delta": 1590950912, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 111432171520, + "reserved_bytes_current_delta": 10636754944, + "reserved_bytes_peak": 111432171520, + "reserved_bytes_peak_delta": 10636754944, + "segment_count_current": 497, + "segment_count_current_delta": 30, + "segment_count_peak": 497, + "segment_count_peak_delta": 30, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T122347835939Z.pickle", + "timestamp_utc": "2026-08-01T12:23:47.945007Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T122347835939Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T122347835939Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4180a0a1d38af245da988528c74b5bba099a0f69 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T122347835939Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3058efe9689beaef88db321e4a42ad1a8e6fa9891a0097cfd8ba51ca312b18dc +size 6057113 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T122348917338Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T122348917338Z.json new file mode 100644 index 0000000000000000000000000000000000000000..761c6a630653757e95f138225b06c3e322ce6851 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T122348917338Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106077354496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106077354496, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105799068672, + "device": 2, + "inactive_split_bytes_current": 8479923200, + "inactive_split_bytes_peak": 9144410112, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 111432171520, + "reserved_bytes_peak": 111432171520, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T12:23:48.916886Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105799068672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106077354496, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105799068672, + "device": 2, + "inactive_split_bytes_current": 7802543104, + "inactive_split_bytes_peak": 9144410112, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 111432171520, + "reserved_bytes_peak": 111432171520, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T12:23:47.958948Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8479923200, + "inactive_split_bytes_current_delta": 677380096, + "inactive_split_bytes_peak": 9144410112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 111432171520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111432171520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 497, + "segment_count_current_delta": 0, + "segment_count_peak": 497, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T122348917338Z.pickle", + "timestamp_utc": "2026-08-01T12:23:49.023220Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T122348917338Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T122348917338Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f96bacba2d9f4040037f430842e7412db0762c8b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T122348917338Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2603797e23a0a600b06a70ec18e1c623b1ccfd0caf2e66459396a9ee7e138104 +size 6587396 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T122350816459Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T122350816459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..35f6d22083315e1f0e823b77c5773bbcd4fe1fea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T122350816459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 141777122304, + "active_bytes_current_delta": 69319561216, + "active_bytes_peak": 163346589696, + "active_bytes_peak_delta": 57269235200, + "after": { + "active_bytes_current": 141777122304, + "active_bytes_peak": 163346589696, + "allocated_bytes_current": 72474604544, + "allocated_bytes_peak": 105807643648, + "device": 2, + "inactive_split_bytes_current": 12338383872, + "inactive_split_bytes_peak": 15608781824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 182928277504, + "reserved_bytes_peak": 182928277504, + "segment_count_current": 808, + "segment_count_peak": 808, + "timestamp_utc": "2026-08-01T12:23:50.815892Z", + "world_size": 8 + }, + "allocated_bytes_current": 72474604544, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 105807643648, + "allocated_bytes_peak_delta": 8574976, + "before": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106077354496, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105799068672, + "device": 2, + "inactive_split_bytes_current": 8479923200, + "inactive_split_bytes_peak": 9144410112, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 111432171520, + "reserved_bytes_peak": 111432171520, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T12:23:49.036692Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12338383872, + "inactive_split_bytes_current_delta": 3858460672, + "inactive_split_bytes_peak": 15608781824, + "inactive_split_bytes_peak_delta": 6464371712, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 182928277504, + "reserved_bytes_current_delta": 71496105984, + "reserved_bytes_peak": 182928277504, + "reserved_bytes_peak_delta": 71496105984, + "segment_count_current": 808, + "segment_count_current_delta": 311, + "segment_count_peak": 808, + "segment_count_peak_delta": 311, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T122350816459Z.pickle", + "timestamp_utc": "2026-08-01T12:23:50.934903Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T122350816459Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T122350816459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..53362a99e1dc090aeda8a0102f371debcb5d5b6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T122350816459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:581bc86f0b256dcbbfd74ba842ee30ee3572dda6b8910f1adde2395babc2f7d2 +size 6874909 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T122346775796Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T122346775796Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3eb907e2a8981d225813cceac7239ff5297fb374 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T122346775796Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 37226105856, + "active_bytes_peak": 82437989376, + "active_bytes_peak_delta": 46971653120, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 82437989376, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 82159443968, + "device": 3, + "inactive_split_bytes_current": 6626040832, + "inactive_split_bytes_peak": 8249350144, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 101265178624, + "reserved_bytes_peak": 101265178624, + "segment_count_current": 470, + "segment_count_peak": 470, + "timestamp_utc": "2026-08-01T12:23:46.775193Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 37226105856, + "allocated_bytes_peak": 82159443968, + "allocated_bytes_peak_delta": 46693107712, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:22:38.711629Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6626040832, + "inactive_split_bytes_current_delta": 6493221888, + "inactive_split_bytes_peak": 8249350144, + "inactive_split_bytes_peak_delta": 8088964096, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 101265178624, + "reserved_bytes_current_delta": 62021173248, + "reserved_bytes_peak": 101265178624, + "reserved_bytes_peak_delta": 62021173248, + "segment_count_current": 470, + "segment_count_current_delta": 399, + "segment_count_peak": 470, + "segment_count_peak_delta": 399, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T122346775796Z.pickle", + "timestamp_utc": "2026-08-01T12:23:46.896793Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T122346775796Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T122346775796Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb92be049db54c584b228b6c84ae4dbe2583022c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T122346775796Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:122674b0a40a0e4aa2040f99149213a8d3dc79320a8487c3e12967f3036cbd10 +size 5943117 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T122347835808Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T122347835808Z.json new file mode 100644 index 0000000000000000000000000000000000000000..13cedbe56e8e19e2f3440416934b669199172909 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T122347835808Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106075599360, + "active_bytes_peak_delta": 23637609984, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106075599360, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105797053952, + "device": 3, + "inactive_split_bytes_current": 7334878208, + "inactive_split_bytes_peak": 8552599552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 111664955392, + "reserved_bytes_peak": 111664955392, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T12:23:47.835406Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105797053952, + "allocated_bytes_peak_delta": 23637609984, + "before": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 82437989376, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 82159443968, + "device": 3, + "inactive_split_bytes_current": 6626040832, + "inactive_split_bytes_peak": 8249350144, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 101265178624, + "reserved_bytes_peak": 101265178624, + "segment_count_current": 470, + "segment_count_peak": 470, + "timestamp_utc": "2026-08-01T12:23:46.915046Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7334878208, + "inactive_split_bytes_current_delta": 708837376, + "inactive_split_bytes_peak": 8552599552, + "inactive_split_bytes_peak_delta": 303249408, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 111664955392, + "reserved_bytes_current_delta": 10399776768, + "reserved_bytes_peak": 111664955392, + "reserved_bytes_peak_delta": 10399776768, + "segment_count_current": 497, + "segment_count_current_delta": 27, + "segment_count_peak": 497, + "segment_count_peak_delta": 27, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T122347835808Z.pickle", + "timestamp_utc": "2026-08-01T12:23:47.942495Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T122347835808Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T122347835808Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3800e241fab374493dbe33645be573a5d4209b11 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T122347835808Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9da2051d420cf20d6032ae123f9d380ecf64790a7a4fb13e4ed3a3381f6e0e8d +size 6068150 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T122348917255Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T122348917255Z.json new file mode 100644 index 0000000000000000000000000000000000000000..28dfa9d1673a308451d7c4d639cea3c754a5f068 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T122348917255Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106076979712, + "active_bytes_peak_delta": 1380352, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106076979712, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105799482880, + "device": 3, + "inactive_split_bytes_current": 8270208000, + "inactive_split_bytes_peak": 8552599552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 111664955392, + "reserved_bytes_peak": 111664955392, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T12:23:48.916858Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105799482880, + "allocated_bytes_peak_delta": 2428928, + "before": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106075599360, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105797053952, + "device": 3, + "inactive_split_bytes_current": 7334878208, + "inactive_split_bytes_peak": 8552599552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 111664955392, + "reserved_bytes_peak": 111664955392, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T12:23:47.956819Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8270208000, + "inactive_split_bytes_current_delta": 935329792, + "inactive_split_bytes_peak": 8552599552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 111664955392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111664955392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 497, + "segment_count_current_delta": 0, + "segment_count_peak": 497, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T122348917255Z.pickle", + "timestamp_utc": "2026-08-01T12:23:49.023119Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T122348917255Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T122348917255Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..984d311fac6b4b76495c4621fa13c72ace025902 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T122348917255Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c923778689f8ebfda8fd49ccfba1b07ed1bb27b2906e2dcc742512d1308ca9d +size 6587662 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T122346775654Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T122346775654Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e9b64f34bfce8a830c34c6fe9ab886fadc412c7b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T122346775654Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459094016, + "active_bytes_current_delta": 37227196416, + "active_bytes_peak": 82502590464, + "active_bytes_peak_delta": 47035811840, + "after": { + "active_bytes_current": 72459094016, + "active_bytes_peak": 82502590464, + "allocated_bytes_current": 72459094016, + "allocated_bytes_peak": 82158396416, + "device": 4, + "inactive_split_bytes_current": 7075395584, + "inactive_split_bytes_peak": 7875206144, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 101470699520, + "reserved_bytes_peak": 101470699520, + "segment_count_current": 470, + "segment_count_peak": 470, + "timestamp_utc": "2026-08-01T12:23:46.775080Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459094016, + "allocated_bytes_current_delta": 37227196416, + "allocated_bytes_peak": 82158396416, + "allocated_bytes_peak_delta": 46691617792, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:22:38.711248Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7075395584, + "inactive_split_bytes_current_delta": 6943019008, + "inactive_split_bytes_peak": 7875206144, + "inactive_split_bytes_peak_delta": 7721796608, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 101470699520, + "reserved_bytes_current_delta": 62226694144, + "reserved_bytes_peak": 101470699520, + "reserved_bytes_peak_delta": 62226694144, + "segment_count_current": 470, + "segment_count_current_delta": 399, + "segment_count_peak": 470, + "segment_count_peak_delta": 399, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T122346775654Z.pickle", + "timestamp_utc": "2026-08-01T12:23:46.894960Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T122346775654Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T122346775654Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..165b034bbd6e911797f867edd1396b1e17a0cf91 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T122346775654Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37bd89e14b9a17011021304c6d2b20e8dd31b274c5c4ec10eef31ceb2559660a +size 5951662 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T122347835809Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T122347835809Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a0b3cf31fdd79852a330bf1ff7328c325cfb7cc4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T122347835809Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459094016, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106076107776, + "active_bytes_peak_delta": 23573517312, + "after": { + "active_bytes_current": 72459094016, + "active_bytes_peak": 106076107776, + "allocated_bytes_current": 72459094016, + "allocated_bytes_peak": 105797562368, + "device": 4, + "inactive_split_bytes_current": 7660500992, + "inactive_split_bytes_peak": 8020389888, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 111165833216, + "reserved_bytes_peak": 111165833216, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T12:23:47.835400Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459094016, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105797562368, + "allocated_bytes_peak_delta": 23639165952, + "before": { + "active_bytes_current": 72459094016, + "active_bytes_peak": 82502590464, + "allocated_bytes_current": 72459094016, + "allocated_bytes_peak": 82158396416, + "device": 4, + "inactive_split_bytes_current": 7075395584, + "inactive_split_bytes_peak": 7875206144, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 101470699520, + "reserved_bytes_peak": 101470699520, + "segment_count_current": 470, + "segment_count_peak": 470, + "timestamp_utc": "2026-08-01T12:23:46.914109Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7660500992, + "inactive_split_bytes_current_delta": 585105408, + "inactive_split_bytes_peak": 8020389888, + "inactive_split_bytes_peak_delta": 145183744, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 111165833216, + "reserved_bytes_current_delta": 9695133696, + "reserved_bytes_peak": 111165833216, + "reserved_bytes_peak_delta": 9695133696, + "segment_count_current": 495, + "segment_count_current_delta": 25, + "segment_count_peak": 495, + "segment_count_peak_delta": 25, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T122347835809Z.pickle", + "timestamp_utc": "2026-08-01T12:23:47.952067Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T122347835809Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T122347835809Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea262827badbb328d8b8896c149411629d8b14ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T122347835809Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e859ebfcbe229467fa173db544831e22e23f8c6aab3f575c8bb6007c64e0a4b +size 6594155 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T122348917307Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T122348917307Z.json new file mode 100644 index 0000000000000000000000000000000000000000..967ce2262d1843cdfe51114eb7bab57313ade6d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T122348917307Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72459094016, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106076107776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72459094016, + "active_bytes_peak": 106076107776, + "allocated_bytes_current": 72459094016, + "allocated_bytes_peak": 105797562368, + "device": 4, + "inactive_split_bytes_current": 7140407296, + "inactive_split_bytes_peak": 8020389888, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 111165833216, + "reserved_bytes_peak": 111165833216, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T12:23:48.916898Z", + "world_size": 8 + }, + "allocated_bytes_current": 72459094016, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105797562368, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72459094016, + "active_bytes_peak": 106076107776, + "allocated_bytes_current": 72459094016, + "allocated_bytes_peak": 105797562368, + "device": 4, + "inactive_split_bytes_current": 7660500992, + "inactive_split_bytes_peak": 8020389888, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 111165833216, + "reserved_bytes_peak": 111165833216, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T12:23:47.968673Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7140407296, + "inactive_split_bytes_current_delta": -520093696, + "inactive_split_bytes_peak": 8020389888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 111165833216, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111165833216, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 495, + "segment_count_current_delta": 0, + "segment_count_peak": 495, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T122348917307Z.pickle", + "timestamp_utc": "2026-08-01T12:23:49.027548Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T122348917307Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T122348917307Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..38431af11633ebb0fa2e50c63ae9fcae30fcb4ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T122348917307Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b304081ba3525fe7d3542871bfc21d5cfebc95c5f6f4f2ba2e14eb18e9f27323 +size 6588001 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T122346779769Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T122346779769Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca056e79379d528dd0b741b37b01366908ee3214 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T122346779769Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 37226105856, + "active_bytes_peak": 82453176320, + "active_bytes_peak_delta": 46986840064, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 82453176320, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 82159443968, + "device": 5, + "inactive_split_bytes_current": 7330683904, + "inactive_split_bytes_peak": 7948107264, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 101382619136, + "reserved_bytes_peak": 101382619136, + "segment_count_current": 470, + "segment_count_peak": 470, + "timestamp_utc": "2026-08-01T12:23:46.779038Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 37226105856, + "allocated_bytes_peak": 82159443968, + "allocated_bytes_peak_delta": 46693107712, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:22:38.711894Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7330683904, + "inactive_split_bytes_current_delta": 7197864960, + "inactive_split_bytes_peak": 7948107264, + "inactive_split_bytes_peak_delta": 7787721216, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 101382619136, + "reserved_bytes_current_delta": 62138613760, + "reserved_bytes_peak": 101382619136, + "reserved_bytes_peak_delta": 62138613760, + "segment_count_current": 470, + "segment_count_current_delta": 399, + "segment_count_peak": 470, + "segment_count_peak_delta": 399, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T122346779769Z.pickle", + "timestamp_utc": "2026-08-01T12:23:46.898823Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T122346779769Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T122346779769Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..831ab1c60ce73fc7499965cf36765ee5ecc03fec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T122346779769Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b6e448ac811141192cdd7be99212de2b493ba5be0dbde3965f5a49cd47f231a +size 5943158 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T122347835852Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T122347835852Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ccb024905160162b8687934a63ae759f50f05dd1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T122347835852Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106077635072, + "active_bytes_peak_delta": 23624458752, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106077635072, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105799220736, + "device": 5, + "inactive_split_bytes_current": 8865799168, + "inactive_split_bytes_peak": 9218120704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 110607990784, + "reserved_bytes_peak": 110607990784, + "segment_count_current": 494, + "segment_count_peak": 494, + "timestamp_utc": "2026-08-01T12:23:47.835439Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105799220736, + "allocated_bytes_peak_delta": 23639776768, + "before": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 82453176320, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 82159443968, + "device": 5, + "inactive_split_bytes_current": 7330683904, + "inactive_split_bytes_peak": 7948107264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 101382619136, + "reserved_bytes_peak": 101382619136, + "segment_count_current": 470, + "segment_count_peak": 470, + "timestamp_utc": "2026-08-01T12:23:46.919904Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8865799168, + "inactive_split_bytes_current_delta": 1535115264, + "inactive_split_bytes_peak": 9218120704, + "inactive_split_bytes_peak_delta": 1270013440, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 110607990784, + "reserved_bytes_current_delta": 9225371648, + "reserved_bytes_peak": 110607990784, + "reserved_bytes_peak_delta": 9225371648, + "segment_count_current": 494, + "segment_count_current_delta": 24, + "segment_count_peak": 494, + "segment_count_peak_delta": 24, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T122347835852Z.pickle", + "timestamp_utc": "2026-08-01T12:23:47.946120Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T122347835852Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T122347835852Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..69d53915c9144b274f8883e17719e03afb87800f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T122347835852Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0190770064e3ec00140c6a36e02ef249e07736ef656c1be0b3b8c969c5c1ac4 +size 6590206 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T122348917245Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T122348917245Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2154e214a1d5191e2bf14e2058695bc28141bfd3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T122348917245Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106078401024, + "active_bytes_peak_delta": 765952, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106078401024, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105799855616, + "device": 5, + "inactive_split_bytes_current": 7334878208, + "inactive_split_bytes_peak": 9218120704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 111021129728, + "reserved_bytes_peak": 111021129728, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T12:23:48.916836Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105799855616, + "allocated_bytes_peak_delta": 634880, + "before": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106077635072, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105799220736, + "device": 5, + "inactive_split_bytes_current": 8865799168, + "inactive_split_bytes_peak": 9218120704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 110607990784, + "reserved_bytes_peak": 110607990784, + "segment_count_current": 494, + "segment_count_peak": 494, + "timestamp_utc": "2026-08-01T12:23:47.962479Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7334878208, + "inactive_split_bytes_current_delta": -1530920960, + "inactive_split_bytes_peak": 9218120704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 111021129728, + "reserved_bytes_current_delta": 413138944, + "reserved_bytes_peak": 111021129728, + "reserved_bytes_peak_delta": 413138944, + "segment_count_current": 495, + "segment_count_current_delta": 1, + "segment_count_peak": 495, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T122348917245Z.pickle", + "timestamp_utc": "2026-08-01T12:23:49.026089Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T122348917245Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T122348917245Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ecac6f9c854cd58c7d184fbbd0924da5c088775 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T122348917245Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9576eb6c06eb80a08ee31a3eb9eac091397ca3988e6a7dd150523bab17d2e6a +size 6576153 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T122346775716Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T122346775716Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b576870bfb5e81c1aed79b773701225b0ea18f2a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T122346775716Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 37226105856, + "active_bytes_peak": 82434291712, + "active_bytes_peak_delta": 46967955456, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 82434291712, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 82158526464, + "device": 6, + "inactive_split_bytes_current": 7095802880, + "inactive_split_bytes_peak": 7605524480, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 100795416576, + "reserved_bytes_peak": 100795416576, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T12:23:46.775105Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 37226105856, + "allocated_bytes_peak": 82158526464, + "allocated_bytes_peak_delta": 46692190208, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:22:38.711640Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7095802880, + "inactive_split_bytes_current_delta": 6962983936, + "inactive_split_bytes_peak": 7605524480, + "inactive_split_bytes_peak_delta": 7445138432, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 100795416576, + "reserved_bytes_current_delta": 61551411200, + "reserved_bytes_peak": 100795416576, + "reserved_bytes_peak_delta": 61551411200, + "segment_count_current": 466, + "segment_count_current_delta": 395, + "segment_count_peak": 466, + "segment_count_peak_delta": 395, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T122346775716Z.pickle", + "timestamp_utc": "2026-08-01T12:23:46.893432Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T122346775716Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T122346775716Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6a0c0a93db6ae65f4472bb64ffa14598432d271b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T122346775716Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b5cbc22abe990bbff228909e6039b1502bd744af8a2226846ef16db34d7ebbd +size 5942516 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T122347835849Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T122347835849Z.json new file mode 100644 index 0000000000000000000000000000000000000000..05ee71b707f195fff6caebc1db5b0218875819b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T122347835849Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106076434944, + "active_bytes_peak_delta": 23642143232, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106076434944, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105797889536, + "device": 6, + "inactive_split_bytes_current": 8094047232, + "inactive_split_bytes_peak": 8799404032, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 111077752832, + "reserved_bytes_peak": 111077752832, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T12:23:47.835456Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105797889536, + "allocated_bytes_peak_delta": 23639363072, + "before": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 82434291712, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 82158526464, + "device": 6, + "inactive_split_bytes_current": 7095802880, + "inactive_split_bytes_peak": 7605524480, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 100795416576, + "reserved_bytes_peak": 100795416576, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T12:23:46.912426Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8094047232, + "inactive_split_bytes_current_delta": 998244352, + "inactive_split_bytes_peak": 8799404032, + "inactive_split_bytes_peak_delta": 1193879552, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 111077752832, + "reserved_bytes_current_delta": 10282336256, + "reserved_bytes_peak": 111077752832, + "reserved_bytes_peak_delta": 10282336256, + "segment_count_current": 493, + "segment_count_current_delta": 27, + "segment_count_peak": 493, + "segment_count_peak_delta": 27, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T122347835849Z.pickle", + "timestamp_utc": "2026-08-01T12:23:47.942553Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T122347835849Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T122347835849Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8bee789151a93f6fbd130d5159e12a6ed058b936 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T122347835849Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:912470ee16460fa72f2ad5fd4b5ccbe8b9450634c420afe0664638c397d3a8f7 +size 6055236 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T122348917256Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T122348917256Z.json new file mode 100644 index 0000000000000000000000000000000000000000..71938cc6ba557699454a7a3907bd5faa392b39b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T122348917256Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106080911872, + "active_bytes_peak_delta": 4476928, + "after": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106080911872, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105802366464, + "device": 6, + "inactive_split_bytes_current": 7597022208, + "inactive_split_bytes_peak": 8830254080, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 111490891776, + "reserved_bytes_peak": 111490891776, + "segment_count_current": 494, + "segment_count_peak": 494, + "timestamp_utc": "2026-08-01T12:23:48.916847Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105802366464, + "allocated_bytes_peak_delta": 4476928, + "before": { + "active_bytes_current": 72457561088, + "active_bytes_peak": 106076434944, + "allocated_bytes_current": 72457561088, + "allocated_bytes_peak": 105797889536, + "device": 6, + "inactive_split_bytes_current": 8094047232, + "inactive_split_bytes_peak": 8799404032, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 111077752832, + "reserved_bytes_peak": 111077752832, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T12:23:47.956973Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7597022208, + "inactive_split_bytes_current_delta": -497025024, + "inactive_split_bytes_peak": 8830254080, + "inactive_split_bytes_peak_delta": 30850048, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 111490891776, + "reserved_bytes_current_delta": 413138944, + "reserved_bytes_peak": 111490891776, + "reserved_bytes_peak_delta": 413138944, + "segment_count_current": 494, + "segment_count_current_delta": 1, + "segment_count_peak": 494, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T122348917256Z.pickle", + "timestamp_utc": "2026-08-01T12:23:49.022206Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T122348917256Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T122348917256Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93ffc86efe0e5dfc01762802a78b42a370185abc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T122348917256Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa57eafe6f17ad85a1403d180c7d502b0969afe308e96a91062594e4f5cb69ac +size 6066088 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T122346775649Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T122346775649Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3075dbf7d56bb1984bdef1fe6700d05550b3097 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T122346775649Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561600, + "active_bytes_current_delta": 37226106368, + "active_bytes_peak": 82437071872, + "active_bytes_peak_delta": 46970735616, + "after": { + "active_bytes_current": 72457561600, + "active_bytes_peak": 82437071872, + "allocated_bytes_current": 72457561600, + "allocated_bytes_peak": 82158526464, + "device": 7, + "inactive_split_bytes_current": 7213242880, + "inactive_split_bytes_peak": 8168795136, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 100912857088, + "reserved_bytes_peak": 100912857088, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:23:46.775084Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561600, + "allocated_bytes_current_delta": 37226106368, + "allocated_bytes_peak": 82158526464, + "allocated_bytes_peak_delta": 46692190208, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:22:38.711689Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7213242880, + "inactive_split_bytes_current_delta": 7080423936, + "inactive_split_bytes_peak": 8168795136, + "inactive_split_bytes_peak_delta": 8008409088, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 100912857088, + "reserved_bytes_current_delta": 61668851712, + "reserved_bytes_peak": 100912857088, + "reserved_bytes_peak_delta": 61668851712, + "segment_count_current": 468, + "segment_count_current_delta": 397, + "segment_count_peak": 468, + "segment_count_peak_delta": 397, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T122346775649Z.pickle", + "timestamp_utc": "2026-08-01T12:23:46.891554Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T122346775649Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T122346775649Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d919b4f13b65922aa57fe64ba49fe4f8faffdfd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T122346775649Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dce69cfed2a87a3998d17b851cb89268c7c389c5dd518ca06e11743b281d448 +size 5944402 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T122347835906Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T122347835906Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2bd4cc5915ef61db377ed8cb81e258bac00238c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T122347835906Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106080105984, + "active_bytes_peak_delta": 23643034112, + "after": { + "active_bytes_current": 72457561600, + "active_bytes_peak": 106080105984, + "allocated_bytes_current": 72457561600, + "allocated_bytes_peak": 105801560576, + "device": 7, + "inactive_split_bytes_current": 9098582528, + "inactive_split_bytes_peak": 9450904064, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112369598464, + "reserved_bytes_peak": 112369598464, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T12:23:47.835509Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105801560576, + "allocated_bytes_peak_delta": 23643034112, + "before": { + "active_bytes_current": 72457561600, + "active_bytes_peak": 82437071872, + "allocated_bytes_current": 72457561600, + "allocated_bytes_peak": 82158526464, + "device": 7, + "inactive_split_bytes_current": 7213242880, + "inactive_split_bytes_peak": 8168795136, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 100912857088, + "reserved_bytes_peak": 100912857088, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:23:46.906874Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9098582528, + "inactive_split_bytes_current_delta": 1885339648, + "inactive_split_bytes_peak": 9450904064, + "inactive_split_bytes_peak_delta": 1282108928, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112369598464, + "reserved_bytes_current_delta": 11456741376, + "reserved_bytes_peak": 112369598464, + "reserved_bytes_peak_delta": 11456741376, + "segment_count_current": 498, + "segment_count_current_delta": 30, + "segment_count_peak": 498, + "segment_count_peak_delta": 30, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T122347835906Z.pickle", + "timestamp_utc": "2026-08-01T12:23:47.939899Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T122347835906Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T122347835906Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..28edb926054a0a0c59cd5a6aa1d396ac982a7a83 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T122347835906Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b23c4553763aa4b7d3c739150cab224f5c079550e63ae6950c26cd6e9cb5c3a7 +size 6069905 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T122348920672Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T122348920672Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2cffc5f41b4fbfaef7be95fd3293bcbce9a142f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T122348920672Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72457561600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106080105984, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72457561600, + "active_bytes_peak": 106080105984, + "allocated_bytes_current": 72457561600, + "allocated_bytes_peak": 105801560576, + "device": 7, + "inactive_split_bytes_current": 8154864128, + "inactive_split_bytes_peak": 9450904064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112369598464, + "reserved_bytes_peak": 112369598464, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T12:23:48.916928Z", + "world_size": 8 + }, + "allocated_bytes_current": 72457561600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105801560576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72457561600, + "active_bytes_peak": 106080105984, + "allocated_bytes_current": 72457561600, + "allocated_bytes_peak": 105801560576, + "device": 7, + "inactive_split_bytes_current": 9098582528, + "inactive_split_bytes_peak": 9450904064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112369598464, + "reserved_bytes_peak": 112369598464, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T12:23:47.951948Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8154864128, + "inactive_split_bytes_current_delta": -943718400, + "inactive_split_bytes_peak": 9450904064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112369598464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112369598464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 498, + "segment_count_current_delta": 0, + "segment_count_peak": 498, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T122348920672Z.pickle", + "timestamp_utc": "2026-08-01T12:23:49.028750Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T122348920672Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T122348920672Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ded9c89ab2dc2e4b84e967493a290e20ad967bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T122348920672Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6a32c1b3125c433242c10dc9aef7a3525a2682365f06734161001100eb9f10e +size 6589799 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..5961cb8c53ec7e43a5912af321e7915bb96bdda4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 4096 +capacity: 1.00 +expert_dispatch_overlap: off +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6012 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 241 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T122603660895Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T122603660895Z.json new file mode 100644 index 0000000000000000000000000000000000000000..382f8be2e6274ab82fd79b34380c5fb7ef74a202 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T122603660895Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922290176, + "active_bytes_current_delta": 31690834944, + "active_bytes_peak": 87151730176, + "active_bytes_peak_delta": 51685393920, + "after": { + "active_bytes_current": 66922290176, + "active_bytes_peak": 87151730176, + "allocated_bytes_current": 66922290176, + "allocated_bytes_peak": 87151730176, + "device": 0, + "inactive_split_bytes_current": 5949547520, + "inactive_split_bytes_peak": 9453646848, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 93354721280, + "reserved_bytes_peak": 93354721280, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T12:26:03.660294Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922290176, + "allocated_bytes_current_delta": 31690834944, + "allocated_bytes_peak": 87151730176, + "allocated_bytes_peak_delta": 51685393920, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:24:54.434596Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5949547520, + "inactive_split_bytes_current_delta": 5814631424, + "inactive_split_bytes_peak": 9453646848, + "inactive_split_bytes_peak_delta": 9293015040, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 93354721280, + "reserved_bytes_current_delta": 54108618752, + "reserved_bytes_peak": 93354721280, + "reserved_bytes_peak_delta": 54108618752, + "segment_count_current": 518, + "segment_count_current_delta": 446, + "segment_count_peak": 518, + "segment_count_peak_delta": 446, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T122603660895Z.pickle", + "timestamp_utc": "2026-08-01T12:26:03.770647Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T122603660895Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T122603660895Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c6888c7081e60f4e01b5661e2547185fd0490e8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T122603660895Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:639c72561ff58657433aed0822ece20280d8ad0d712c6983acb55b5c2474a5c5 +size 5394406 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T122604886922Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T122604886922Z.json new file mode 100644 index 0000000000000000000000000000000000000000..507dde3762ff194e6fc8e16ff4586f8d8f78fa04 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T122604886922Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922290176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110780059648, + "active_bytes_peak_delta": 23628329472, + "after": { + "active_bytes_current": 66922290176, + "active_bytes_peak": 110780059648, + "allocated_bytes_current": 66922290176, + "allocated_bytes_peak": 110780059648, + "device": 0, + "inactive_split_bytes_current": 6364783616, + "inactive_split_bytes_peak": 9453646848, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112965189632, + "reserved_bytes_peak": 112965189632, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T12:26:04.886521Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922290176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110780059648, + "allocated_bytes_peak_delta": 23628329472, + "before": { + "active_bytes_current": 66922290176, + "active_bytes_peak": 87151730176, + "allocated_bytes_current": 66922290176, + "allocated_bytes_peak": 87151730176, + "device": 0, + "inactive_split_bytes_current": 5949547520, + "inactive_split_bytes_peak": 9453646848, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 93354721280, + "reserved_bytes_peak": 93354721280, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T12:26:03.789477Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6364783616, + "inactive_split_bytes_current_delta": 415236096, + "inactive_split_bytes_peak": 9453646848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112965189632, + "reserved_bytes_current_delta": 19610468352, + "reserved_bytes_peak": 112965189632, + "reserved_bytes_peak_delta": 19610468352, + "segment_count_current": 603, + "segment_count_current_delta": 85, + "segment_count_peak": 603, + "segment_count_peak_delta": 85, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T122604886922Z.pickle", + "timestamp_utc": "2026-08-01T12:26:04.994998Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T122604886922Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T122604886922Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1a274363f3ad4acc03be6157192ec7b1375204df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T122604886922Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47a4be034610760ce34759c0975c9aed3959ddf05a5e769662909c5d374c6f19 +size 5588681 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T122605993674Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T122605993674Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3420cc34216f208f91139d0574c9a275d0dba52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T122605993674Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922290176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110780059648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66922290176, + "active_bytes_peak": 110780059648, + "allocated_bytes_current": 66922290176, + "allocated_bytes_peak": 110780059648, + "device": 0, + "inactive_split_bytes_current": 6433989632, + "inactive_split_bytes_peak": 9453646848, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112965189632, + "reserved_bytes_peak": 112965189632, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T12:26:05.993132Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922290176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110780059648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66922290176, + "active_bytes_peak": 110780059648, + "allocated_bytes_current": 66922290176, + "allocated_bytes_peak": 110780059648, + "device": 0, + "inactive_split_bytes_current": 6364783616, + "inactive_split_bytes_peak": 9453646848, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112965189632, + "reserved_bytes_peak": 112965189632, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T12:26:05.010218Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6433989632, + "inactive_split_bytes_current_delta": 69206016, + "inactive_split_bytes_peak": 9453646848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112965189632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112965189632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T122605993674Z.pickle", + "timestamp_utc": "2026-08-01T12:26:06.105191Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T122605993674Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T122605993674Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..163ae114e0276aee324774f4a30b24eec2341446 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T122605993674Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cf2686ced17ea7aef31738f602e3712dcbe1ee81fa5db50ad87334f5fa5f7e0 +size 5589353 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T122608842336Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T122608842336Z.json new file mode 100644 index 0000000000000000000000000000000000000000..45dd69b25cabb685561ad280e6f1a2963b8114c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T122608842336Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939333632, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128945801216, + "active_bytes_peak_delta": 18165741568, + "after": { + "active_bytes_current": 66939333632, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66939333632, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_peak": 14005938176, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_peak": 142222557184, + "segment_count_current": 831, + "segment_count_peak": 831, + "timestamp_utc": "2026-08-01T12:26:08.841793Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939333632, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110793091072, + "allocated_bytes_peak_delta": 13031424, + "before": { + "active_bytes_current": 66922290176, + "active_bytes_peak": 110780059648, + "allocated_bytes_current": 66922290176, + "allocated_bytes_peak": 110780059648, + "device": 0, + "inactive_split_bytes_current": 6433989632, + "inactive_split_bytes_peak": 9453646848, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112965189632, + "reserved_bytes_peak": 112965189632, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T12:26:06.121659Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_current_delta": 2872832000, + "inactive_split_bytes_peak": 14005938176, + "inactive_split_bytes_peak_delta": 4552291328, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_current_delta": 29257367552, + "reserved_bytes_peak": 142222557184, + "reserved_bytes_peak_delta": 29257367552, + "segment_count_current": 831, + "segment_count_current_delta": 228, + "segment_count_peak": 831, + "segment_count_peak_delta": 228, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T122608842336Z.pickle", + "timestamp_utc": "2026-08-01T12:26:08.963872Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T122608842336Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T122608842336Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e042ba647cd7a835d34cfee7a84739475968ed7c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T122608842336Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31bbfd63aa0cbc358cc69b808f2782213d0a7580459470db68b5a582b22f8bff +size 6595820 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T122609901050Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T122609901050Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4401887d3c9d8a1f9cbacf4ca74b0650ffb28c57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T122609901050Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939333632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128945801216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939333632, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66939333632, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_peak": 14005938176, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_peak": 142222557184, + "segment_count_current": 831, + "segment_count_peak": 831, + "timestamp_utc": "2026-08-01T12:26:09.900585Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939333632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110793091072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939333632, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66939333632, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_peak": 14005938176, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_peak": 142222557184, + "segment_count_current": 831, + "segment_count_peak": 831, + "timestamp_utc": "2026-08-01T12:26:08.980498Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14005938176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142222557184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 831, + "segment_count_current_delta": 0, + "segment_count_peak": 831, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T122609901050Z.pickle", + "timestamp_utc": "2026-08-01T12:26:09.908511Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T122609901050Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T122609901050Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c47be443d6dc388596cf3134c8b64b1c6ddd28e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T122609901050Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:848c97f7ec19d04dc51cab82b0cad46b02a0e806fd1618913f8e5b649cc3055c +size 270422 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T122610830235Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T122610830235Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a67bea7191c00777ee64db8c57578ad64305d5b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T122610830235Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939333632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128945801216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939333632, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66939333632, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_peak": 14005938176, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_peak": 142222557184, + "segment_count_current": 831, + "segment_count_peak": 831, + "timestamp_utc": "2026-08-01T12:26:10.829837Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939333632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110793091072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939333632, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66939333632, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_peak": 14005938176, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_peak": 142222557184, + "segment_count_current": 831, + "segment_count_peak": 831, + "timestamp_utc": "2026-08-01T12:26:09.909527Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14005938176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142222557184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 831, + "segment_count_current_delta": 0, + "segment_count_peak": 831, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T122610830235Z.pickle", + "timestamp_utc": "2026-08-01T12:26:10.837153Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T122610830235Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T122610830235Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d86c9f6f667f7d83fc89bd3cfff3f2dd531dee00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T122610830235Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d712d72da35181cb04d95f5f788a78e0a9ccb032538377881274a5d512f2c68 +size 270391 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T122611760891Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T122611760891Z.json new file mode 100644 index 0000000000000000000000000000000000000000..55471a4b2fc3d42835e6b349a1fafea15cb4a3f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T122611760891Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939333632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128945801216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939333632, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66939333632, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_peak": 14005938176, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_peak": 142222557184, + "segment_count_current": 831, + "segment_count_peak": 831, + "timestamp_utc": "2026-08-01T12:26:11.760494Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939333632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110793091072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939333632, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66939333632, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_peak": 14005938176, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_peak": 142222557184, + "segment_count_current": 831, + "segment_count_peak": 831, + "timestamp_utc": "2026-08-01T12:26:10.838124Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14005938176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142222557184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 831, + "segment_count_current_delta": 0, + "segment_count_peak": 831, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T122611760891Z.pickle", + "timestamp_utc": "2026-08-01T12:26:11.767441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T122611760891Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T122611760891Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..257b954dcf7ea21cbfff50f3affb108c4c5c2d23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T122611760891Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a53464d9747339740986e29ebe1c798eef2ecb3c2708035126980df73e6d7b6 +size 270391 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T122612685647Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T122612685647Z.json new file mode 100644 index 0000000000000000000000000000000000000000..32ef30f953a29e50eeda2761cb3f4e08ac7a8ce7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T122612685647Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939333632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128945801216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939333632, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66939333632, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_peak": 14005938176, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_peak": 142222557184, + "segment_count_current": 831, + "segment_count_peak": 831, + "timestamp_utc": "2026-08-01T12:26:12.685261Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939333632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110793091072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939333632, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66939333632, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_peak": 14005938176, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_peak": 142222557184, + "segment_count_current": 831, + "segment_count_peak": 831, + "timestamp_utc": "2026-08-01T12:26:11.768426Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14005938176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142222557184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 831, + "segment_count_current_delta": 0, + "segment_count_peak": 831, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T122612685647Z.pickle", + "timestamp_utc": "2026-08-01T12:26:12.692111Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T122612685647Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T122612685647Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4d507ede6ac179a752edd76d6ceaf3d40622828b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T122612685647Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:175fc8afc49d724d1e54f0ebf50868fe87593a91003a2abf49ff9e002d109c00 +size 270422 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T122613615223Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T122613615223Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bc4b011dbc1ef1a4e3494ad9d070a6e341de26ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T122613615223Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939333632, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128945801216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939333632, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66939333632, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_peak": 14005938176, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_peak": 142222557184, + "segment_count_current": 831, + "segment_count_peak": 831, + "timestamp_utc": "2026-08-01T12:26:13.614829Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939333632, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110793091072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939333632, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66939333632, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_peak": 14005938176, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_peak": 142222557184, + "segment_count_current": 831, + "segment_count_peak": 831, + "timestamp_utc": "2026-08-01T12:26:12.693035Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14005938176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142222557184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 831, + "segment_count_current_delta": 0, + "segment_count_peak": 831, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T122613615223Z.pickle", + "timestamp_utc": "2026-08-01T12:26:13.622497Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T122613615223Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T122613615223Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e2e7c78a2bce2598e8db331fca32d42d72d21d37 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T122613615223Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eef017027f34f6fedbb37ddda7b1591225d71e20e3b514a50b7bd970abd5da2 +size 270391 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T122714244921Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T122714244921Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8268ab87b0ee921cafdaa64edc2029c357eb6d4c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T122714244921Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66956260352, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 128945801216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66956260352, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66956260352, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 5458398208, + "inactive_split_bytes_peak": 14005938176, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 143115943936, + "reserved_bytes_peak": 143115943936, + "segment_count_current": 837, + "segment_count_peak": 837, + "timestamp_utc": "2026-08-01T12:27:14.244450Z", + "world_size": 8 + }, + "allocated_bytes_current": 66956260352, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110793091072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939333632, + "active_bytes_peak": 128945801216, + "allocated_bytes_current": 66939333632, + "allocated_bytes_peak": 110793091072, + "device": 0, + "inactive_split_bytes_current": 9306821632, + "inactive_split_bytes_peak": 14005938176, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 142222557184, + "reserved_bytes_peak": 142222557184, + "segment_count_current": 831, + "segment_count_peak": 831, + "timestamp_utc": "2026-08-01T12:26:13.623436Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5458398208, + "inactive_split_bytes_current_delta": -3848423424, + "inactive_split_bytes_peak": 14005938176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 143115943936, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 143115943936, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 837, + "segment_count_current_delta": 6, + "segment_count_peak": 837, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T122714244921Z.pickle", + "timestamp_utc": "2026-08-01T12:27:14.340330Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T122714244921Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T122714244921Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2be034cbc783cafe3c3c62f526862d644a461c05 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T122714244921Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e0a86180518d4034e746bc675073fb772377dce522386e91ff4661c4158166a +size 4356534 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T122603660927Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T122603660927Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2f3a096ee6eb86d7767a9805d9d7c78b406fbf25 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T122603660927Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 87150354432, + "active_bytes_peak_delta": 51684018176, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87150354432, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87150354432, + "device": 1, + "inactive_split_bytes_current": 6613419008, + "inactive_split_bytes_peak": 9121575936, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 94474600448, + "reserved_bytes_peak": 94474600448, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:26:03.660354Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 87150354432, + "allocated_bytes_peak_delta": 51684018176, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:24:54.433949Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6613419008, + "inactive_split_bytes_current_delta": 6480600064, + "inactive_split_bytes_peak": 9121575936, + "inactive_split_bytes_peak_delta": 8961189888, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 94474600448, + "reserved_bytes_current_delta": 55230595072, + "reserved_bytes_peak": 94474600448, + "reserved_bytes_peak_delta": 55230595072, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T122603660927Z.pickle", + "timestamp_utc": "2026-08-01T12:26:03.774266Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T122603660927Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T122603660927Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..88abd62fdd577c2003638bc48baf6c260394c9a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T122603660927Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d19d52f808618a11edca1af194c3cca8e2d02143ab40f9541b1152b73af2d57 +size 5388822 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T122604887015Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T122604887015Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03e2cc57a63c526aa3d5f434d0e77cdf6815a9ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T122604887015Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110778113024, + "active_bytes_peak_delta": 23627758592, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110778113024, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110778113024, + "device": 1, + "inactive_split_bytes_current": 7332742144, + "inactive_split_bytes_peak": 9121575936, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112963092480, + "reserved_bytes_peak": 112963092480, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T12:26:04.886611Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110778113024, + "allocated_bytes_peak_delta": 23627758592, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87150354432, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87150354432, + "device": 1, + "inactive_split_bytes_current": 6613419008, + "inactive_split_bytes_peak": 9121575936, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 94474600448, + "reserved_bytes_peak": 94474600448, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:26:03.793766Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7332742144, + "inactive_split_bytes_current_delta": 719323136, + "inactive_split_bytes_peak": 9121575936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112963092480, + "reserved_bytes_current_delta": 18488492032, + "reserved_bytes_peak": 112963092480, + "reserved_bytes_peak_delta": 18488492032, + "segment_count_current": 591, + "segment_count_current_delta": 79, + "segment_count_peak": 591, + "segment_count_peak_delta": 79, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T122604887015Z.pickle", + "timestamp_utc": "2026-08-01T12:26:04.997731Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T122604887015Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T122604887015Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b407bf5fa4469e2a5f7ec2374effba702716bd14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T122604887015Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddcafeaf6c3fd8d5a2d2922977b2d11ee9654c4bf5a67ce4c9d29c7a674f48e9 +size 5584716 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T122605993528Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T122605993528Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a4ff3ae4e545d789de9d06ab8a82b655237cf15 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T122605993528Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110780829696, + "active_bytes_peak_delta": 2716672, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110780829696, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110780829696, + "device": 1, + "inactive_split_bytes_current": 6623904768, + "inactive_split_bytes_peak": 9121575936, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112963092480, + "reserved_bytes_peak": 112963092480, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T12:26:05.993121Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110780829696, + "allocated_bytes_peak_delta": 2716672, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110778113024, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110778113024, + "device": 1, + "inactive_split_bytes_current": 7332742144, + "inactive_split_bytes_peak": 9121575936, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112963092480, + "reserved_bytes_peak": 112963092480, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T12:26:05.015201Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6623904768, + "inactive_split_bytes_current_delta": -708837376, + "inactive_split_bytes_peak": 9121575936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112963092480, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112963092480, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 591, + "segment_count_current_delta": 0, + "segment_count_peak": 591, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T122605993528Z.pickle", + "timestamp_utc": "2026-08-01T12:26:06.095800Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T122605993528Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T122605993528Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ffa17d497c6c397853fc3f2922b76f3604fd4aba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T122605993528Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:866f78aba9a3d9528a95be3afe2e8a9bc497103f357de9fae42d2c5ff53e4187 +size 5584959 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T122608842494Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T122608842494Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bf1349b7860d9f3a202fb677b7a90729fca11d43 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T122608842494Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 18169042944, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 13913134080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142461632512, + "reserved_bytes_peak": 142461632512, + "segment_count_current": 823, + "segment_count_peak": 823, + "timestamp_utc": "2026-08-01T12:26:08.841881Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 16332800, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110780829696, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110780829696, + "device": 1, + "inactive_split_bytes_current": 6623904768, + "inactive_split_bytes_peak": 9121575936, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112963092480, + "reserved_bytes_peak": 112963092480, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T12:26:06.110359Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_current_delta": 2973495296, + "inactive_split_bytes_peak": 13913134080, + "inactive_split_bytes_peak_delta": 4791558144, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142461632512, + "reserved_bytes_current_delta": 29498540032, + "reserved_bytes_peak": 142461632512, + "reserved_bytes_peak_delta": 29498540032, + "segment_count_current": 823, + "segment_count_current_delta": 232, + "segment_count_peak": 823, + "segment_count_peak_delta": 232, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T122608842494Z.pickle", + "timestamp_utc": "2026-08-01T12:26:08.960923Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T122608842494Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T122608842494Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..943c6d1a1467a70a6132d55f448ae19a852a6082 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T122608842494Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:281471127f75b7e0d3ce13adaff19b62e46e1013ebf6ef1b6d4986e42288f6a3 +size 5859192 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T122609901291Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T122609901291Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d539a0bfcac6c3ea4421df9d41eb5604d9b31eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T122609901291Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 13913134080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142461632512, + "reserved_bytes_peak": 142461632512, + "segment_count_current": 823, + "segment_count_peak": 823, + "timestamp_utc": "2026-08-01T12:26:09.900750Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 13913134080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142461632512, + "reserved_bytes_peak": 142461632512, + "segment_count_current": 823, + "segment_count_peak": 823, + "timestamp_utc": "2026-08-01T12:26:08.977771Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13913134080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142461632512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142461632512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 823, + "segment_count_current_delta": 0, + "segment_count_peak": 823, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T122609901291Z.pickle", + "timestamp_utc": "2026-08-01T12:26:09.909295Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T122609901291Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T122609901291Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7aff493684da08a9b158c51a7392e19d2fc193d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T122609901291Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ecbe8683a2212122cae686a1c849c56f0339de694c06e221a3257c1aa04147f +size 236995 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T122610830378Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T122610830378Z.json new file mode 100644 index 0000000000000000000000000000000000000000..53a7359551a7efad14d98bb80e23b4e46c4c388d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T122610830378Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 13913134080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142461632512, + "reserved_bytes_peak": 142461632512, + "segment_count_current": 823, + "segment_count_peak": 823, + "timestamp_utc": "2026-08-01T12:26:10.829947Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 13913134080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142461632512, + "reserved_bytes_peak": 142461632512, + "segment_count_current": 823, + "segment_count_peak": 823, + "timestamp_utc": "2026-08-01T12:26:09.910276Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13913134080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142461632512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142461632512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 823, + "segment_count_current_delta": 0, + "segment_count_peak": 823, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T122610830378Z.pickle", + "timestamp_utc": "2026-08-01T12:26:10.837144Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T122610830378Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T122610830378Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..86db0e131581d0542a2104dc66d763c7a9a3cc76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T122610830378Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af627fbedbec82bae567c1890f08ec386ec22a3385f2aae43e3c4ab3ba743e6c +size 236995 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T122611761046Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T122611761046Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9492f3baebc032699572068402db8495c934039d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T122611761046Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 13913134080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142461632512, + "reserved_bytes_peak": 142461632512, + "segment_count_current": 823, + "segment_count_peak": 823, + "timestamp_utc": "2026-08-01T12:26:11.760633Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 13913134080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142461632512, + "reserved_bytes_peak": 142461632512, + "segment_count_current": 823, + "segment_count_peak": 823, + "timestamp_utc": "2026-08-01T12:26:10.838102Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13913134080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142461632512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142461632512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 823, + "segment_count_current_delta": 0, + "segment_count_peak": 823, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T122611761046Z.pickle", + "timestamp_utc": "2026-08-01T12:26:11.767530Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T122611761046Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T122611761046Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d06a52045174e364853ec3a7dfa52252320c51c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T122611761046Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19b8e7406ef71f1477ed3cb4da4221b3ddd497aec66141eeaff048448a501863 +size 236995 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T122612685754Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T122612685754Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d1b3d22ade0b1a8394538e13f25191a3fceb93a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T122612685754Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 13913134080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142461632512, + "reserved_bytes_peak": 142461632512, + "segment_count_current": 823, + "segment_count_peak": 823, + "timestamp_utc": "2026-08-01T12:26:12.685357Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 13913134080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142461632512, + "reserved_bytes_peak": 142461632512, + "segment_count_current": 823, + "segment_count_peak": 823, + "timestamp_utc": "2026-08-01T12:26:11.768473Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13913134080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142461632512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142461632512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 823, + "segment_count_current_delta": 0, + "segment_count_peak": 823, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T122612685754Z.pickle", + "timestamp_utc": "2026-08-01T12:26:12.691997Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T122612685754Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T122612685754Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d4050dd671b39cadb8390486831165cfeb7756e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T122612685754Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cde7b49cf6689a84a60809c2938fc9df67828af70be9ab321eee23d8d4eb6ceb +size 236995 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T122613615269Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T122613615269Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3c254f771982cc6dfd4fa31eb28c4b5136dab1c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T122613615269Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 13913134080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142461632512, + "reserved_bytes_peak": 142461632512, + "segment_count_current": 823, + "segment_count_peak": 823, + "timestamp_utc": "2026-08-01T12:26:13.614880Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 13913134080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142461632512, + "reserved_bytes_peak": 142461632512, + "segment_count_current": 823, + "segment_count_peak": 823, + "timestamp_utc": "2026-08-01T12:26:12.692942Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13913134080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142461632512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142461632512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 823, + "segment_count_current_delta": 0, + "segment_count_peak": 823, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T122613615269Z.pickle", + "timestamp_utc": "2026-08-01T12:26:13.621483Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T122613615269Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T122613615269Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b219109271f48abd8ebb2f5502f28d41071f48f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T122613615269Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9320f035e9c2110748d0e5f10001cc05b9941067b23cf7385cdecca36ff1e9fb +size 236995 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T122714244895Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T122714244895Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a21abf667a72ad2eceb83384d2b1efaaf39b27e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T122714244895Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 5507804160, + "inactive_split_bytes_peak": 13913134080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 143355019264, + "reserved_bytes_peak": 143355019264, + "segment_count_current": 829, + "segment_count_peak": 829, + "timestamp_utc": "2026-08-01T12:27:14.244422Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 1, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 13913134080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142461632512, + "reserved_bytes_peak": 142461632512, + "segment_count_current": 823, + "segment_count_peak": 823, + "timestamp_utc": "2026-08-01T12:26:13.622392Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5507804160, + "inactive_split_bytes_current_delta": -4089595904, + "inactive_split_bytes_peak": 13913134080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 143355019264, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 143355019264, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 829, + "segment_count_current_delta": 6, + "segment_count_peak": 829, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T122714244895Z.pickle", + "timestamp_utc": "2026-08-01T12:27:14.331349Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T122714244895Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T122714244895Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1eb48f29d9d5457d520a412b585c9dc749f9d077 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T122714244895Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6db2efde4834101a5045b01de39809f0ff83d5948e3de1846c94190c1b2f4fd4 +size 4208297 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T122603660822Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T122603660822Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1bdb11607df4e2c559d0ec78223bc2cc5580ce9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T122603660822Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 87150354432, + "active_bytes_peak_delta": 51684018176, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87150354432, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87150354432, + "device": 2, + "inactive_split_bytes_current": 6613419008, + "inactive_split_bytes_peak": 8999859200, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 94476697600, + "reserved_bytes_peak": 94476697600, + "segment_count_current": 513, + "segment_count_peak": 513, + "timestamp_utc": "2026-08-01T12:26:03.660210Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 87150354432, + "allocated_bytes_peak_delta": 51684018176, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:24:54.433671Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6613419008, + "inactive_split_bytes_current_delta": 6480600064, + "inactive_split_bytes_peak": 8999859200, + "inactive_split_bytes_peak_delta": 8839473152, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 94476697600, + "reserved_bytes_current_delta": 55232692224, + "reserved_bytes_peak": 94476697600, + "reserved_bytes_peak_delta": 55232692224, + "segment_count_current": 513, + "segment_count_current_delta": 442, + "segment_count_peak": 513, + "segment_count_peak_delta": 442, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T122603660822Z.pickle", + "timestamp_utc": "2026-08-01T12:26:03.769643Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T122603660822Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T122603660822Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a7a1188686cf0287ad1f35f9ee391ad7656f5313 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T122603660822Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1804903ebddb02bd7c333e769ce7b91a4c4072af0950781398e369490a4d2ee0 +size 5388665 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T122604886894Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T122604886894Z.json new file mode 100644 index 0000000000000000000000000000000000000000..97ead7fa2e27d8718bc3a9684c587fa4da90b294 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T122604886894Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110778113024, + "active_bytes_peak_delta": 23627758592, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110778113024, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110778113024, + "device": 2, + "inactive_split_bytes_current": 7441794048, + "inactive_split_bytes_peak": 8999859200, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112965189632, + "reserved_bytes_peak": 112965189632, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:04.886493Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110778113024, + "allocated_bytes_peak_delta": 23627758592, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87150354432, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87150354432, + "device": 2, + "inactive_split_bytes_current": 6613419008, + "inactive_split_bytes_peak": 8999859200, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 94476697600, + "reserved_bytes_peak": 94476697600, + "segment_count_current": 513, + "segment_count_peak": 513, + "timestamp_utc": "2026-08-01T12:26:03.787235Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7441794048, + "inactive_split_bytes_current_delta": 828375040, + "inactive_split_bytes_peak": 8999859200, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112965189632, + "reserved_bytes_current_delta": 18488492032, + "reserved_bytes_peak": 112965189632, + "reserved_bytes_peak_delta": 18488492032, + "segment_count_current": 592, + "segment_count_current_delta": 79, + "segment_count_peak": 592, + "segment_count_peak_delta": 79, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T122604886894Z.pickle", + "timestamp_utc": "2026-08-01T12:26:04.990297Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T122604886894Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T122604886894Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3cda233c6537b6ce73c922fca453291ed8694589 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T122604886894Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67839c7276f8d5833a9dc3bf4eddc1fb1788362effb9a57f23e3c499a47598de +size 5584443 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T122605993404Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T122605993404Z.json new file mode 100644 index 0000000000000000000000000000000000000000..43af55abb69c53bf0fc49be3aadd68c017525c24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T122605993404Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110780829696, + "active_bytes_peak_delta": 2716672, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110780829696, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110780829696, + "device": 2, + "inactive_split_bytes_current": 6626001920, + "inactive_split_bytes_peak": 8999859200, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112965189632, + "reserved_bytes_peak": 112965189632, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:05.992991Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110780829696, + "allocated_bytes_peak_delta": 2716672, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110778113024, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110778113024, + "device": 2, + "inactive_split_bytes_current": 7441794048, + "inactive_split_bytes_peak": 8999859200, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112965189632, + "reserved_bytes_peak": 112965189632, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:05.003894Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6626001920, + "inactive_split_bytes_current_delta": -815792128, + "inactive_split_bytes_peak": 8999859200, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112965189632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112965189632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T122605993404Z.pickle", + "timestamp_utc": "2026-08-01T12:26:06.092782Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T122605993404Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T122605993404Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..feb4a5c37b6d4ae07aa3a0d3c6df8ec1309a1744 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T122605993404Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e281aeae452cdfe594be68089899c5b2c2375e1867ed30369ea6a4fedf4f193 +size 5585026 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T122608842317Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T122608842317Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3042d1c8d70c2ae150fa39721ebd5308e9bb240b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T122608842317Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 18169042944, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_peak": 13915231232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142463729664, + "reserved_bytes_peak": 142463729664, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:08.841779Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 16332800, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110780829696, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110780829696, + "device": 2, + "inactive_split_bytes_current": 6626001920, + "inactive_split_bytes_peak": 8999859200, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112965189632, + "reserved_bytes_peak": 112965189632, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:06.105706Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_current_delta": 2979786752, + "inactive_split_bytes_peak": 13915231232, + "inactive_split_bytes_peak_delta": 4915372032, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142463729664, + "reserved_bytes_current_delta": 29498540032, + "reserved_bytes_peak": 142463729664, + "reserved_bytes_peak_delta": 29498540032, + "segment_count_current": 824, + "segment_count_current_delta": 232, + "segment_count_peak": 824, + "segment_count_peak_delta": 232, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T122608842317Z.pickle", + "timestamp_utc": "2026-08-01T12:26:08.960620Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T122608842317Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T122608842317Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..83f80d5a2d3a2af62b7dcae859fe68015332db73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T122608842317Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa25891f083aeddb61dbf3df27f251d50b3a94468b692df83bdb713c036d2dec +size 5857317 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T122609901043Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T122609901043Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0fe66714b1939ab066fcde55228f413a2e4b62a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T122609901043Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_peak": 13915231232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142463729664, + "reserved_bytes_peak": 142463729664, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:09.900566Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_peak": 13915231232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142463729664, + "reserved_bytes_peak": 142463729664, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:08.975182Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13915231232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142463729664, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142463729664, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T122609901043Z.pickle", + "timestamp_utc": "2026-08-01T12:26:09.908383Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T122609901043Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T122609901043Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d67ae12fde6e0d77cd74b3c9dfabdabcc971c621 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T122609901043Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b25b0fe8777bf020900fb3a529ec6bf86268ad19c8cee5539baab97ca7b04c53 +size 235352 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T122610830255Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T122610830255Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8feebe20c57146eb97960205438a8820af074193 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T122610830255Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_peak": 13915231232, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142463729664, + "reserved_bytes_peak": 142463729664, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:10.829836Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_peak": 13915231232, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142463729664, + "reserved_bytes_peak": 142463729664, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:09.909365Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13915231232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142463729664, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142463729664, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T122610830255Z.pickle", + "timestamp_utc": "2026-08-01T12:26:10.841476Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T122610830255Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T122610830255Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c77ebf25c35c1b5c4f49164d1233b0f2061c2156 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T122610830255Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a77d33685fcd9ef5ccefd38a597df4a5d988e9c036e45eb170676b7d47b88dc0 +size 235352 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T122611760926Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T122611760926Z.json new file mode 100644 index 0000000000000000000000000000000000000000..737e791837f37dcd7e6c5883c9d717d43fff8033 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T122611760926Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_peak": 13915231232, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142463729664, + "reserved_bytes_peak": 142463729664, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:11.760516Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_peak": 13915231232, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142463729664, + "reserved_bytes_peak": 142463729664, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:10.842575Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13915231232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142463729664, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142463729664, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T122611760926Z.pickle", + "timestamp_utc": "2026-08-01T12:26:11.767830Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T122611760926Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T122611760926Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2edac0cf03e5d7517f4b9a095e8261f3f81b77fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T122611760926Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18c99b8c711f99b34a536a9518e229ee6be666b42cc8c279f150d238923848b1 +size 235352 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T122612685635Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T122612685635Z.json new file mode 100644 index 0000000000000000000000000000000000000000..da9d26f614477e892bc52a43dbda811dad5b4d4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T122612685635Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_peak": 13915231232, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142463729664, + "reserved_bytes_peak": 142463729664, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:12.685241Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_peak": 13915231232, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142463729664, + "reserved_bytes_peak": 142463729664, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:11.768812Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13915231232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142463729664, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142463729664, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T122612685635Z.pickle", + "timestamp_utc": "2026-08-01T12:26:12.691955Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T122612685635Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T122612685635Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d50a72de6805705354cea3d5f6a60b6a92159652 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T122612685635Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21a53c4583f17c914da73d9c34a2d1d13e2bb16d9b5181c35582c9988fc8b23e +size 235352 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T122613615422Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T122613615422Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65edf771835ac8c36748de22c80377101bf16810 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T122613615422Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_peak": 13915231232, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142463729664, + "reserved_bytes_peak": 142463729664, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:13.614990Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_peak": 13915231232, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142463729664, + "reserved_bytes_peak": 142463729664, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:12.692896Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13915231232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142463729664, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142463729664, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T122613615422Z.pickle", + "timestamp_utc": "2026-08-01T12:26:13.624298Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T122613615422Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T122613615422Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3df71e13bc553b00a3ee22a757b1cea5e3913eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T122613615422Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d102242d50625c551de9feb61724a1bb763bd1bf292d7ab67483ea92332b2ad +size 235352 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T122714244920Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T122714244920Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2caf8805c3007ec2ce8f331bde5d314d181ae4ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T122714244920Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 128949872640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 5507804160, + "inactive_split_bytes_peak": 13915231232, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 143357116416, + "reserved_bytes_peak": 143357116416, + "segment_count_current": 830, + "segment_count_peak": 830, + "timestamp_utc": "2026-08-01T12:27:14.244437Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110797162496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128949872640, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110797162496, + "device": 2, + "inactive_split_bytes_current": 9605788672, + "inactive_split_bytes_peak": 13915231232, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142463729664, + "reserved_bytes_peak": 142463729664, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:13.625279Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5507804160, + "inactive_split_bytes_current_delta": -4097984512, + "inactive_split_bytes_peak": 13915231232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 143357116416, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 143357116416, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 830, + "segment_count_current_delta": 6, + "segment_count_peak": 830, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T122714244920Z.pickle", + "timestamp_utc": "2026-08-01T12:27:14.331270Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T122714244920Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T122714244920Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b3c01c905c6f53bc5279f60c021221a079a931f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T122714244920Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f9b7fe3f12c1685f5bd218c094a5a26ae1264293de71297cbba1d08556af209 +size 4186430 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T122603660825Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T122603660825Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e42a71c81c5c432bd3c9ed9f6047196cc9fd9c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T122603660825Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 87150354432, + "active_bytes_peak_delta": 51684018176, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87150354432, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87150354432, + "device": 3, + "inactive_split_bytes_current": 6617613312, + "inactive_split_bytes_peak": 9008247808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 94474600448, + "reserved_bytes_peak": 94474600448, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:26:03.660228Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 87150354432, + "allocated_bytes_peak_delta": 51684018176, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:24:54.434669Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6617613312, + "inactive_split_bytes_current_delta": 6484794368, + "inactive_split_bytes_peak": 9008247808, + "inactive_split_bytes_peak_delta": 8847861760, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 94474600448, + "reserved_bytes_current_delta": 55230595072, + "reserved_bytes_peak": 94474600448, + "reserved_bytes_peak_delta": 55230595072, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T122603660825Z.pickle", + "timestamp_utc": "2026-08-01T12:26:03.773330Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T122603660825Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T122603660825Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2fedd0da7d46d740fd2117c0768e02f6c5ff834b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T122603660825Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47856df33d2394b092a11a4d90f7254fc2a2fc7f759818d38a0821224202590e +size 5388934 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T122604886911Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T122604886911Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b29e3ec988068adb96968e2e97e26a3a3335b014 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T122604886911Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110778113024, + "active_bytes_peak_delta": 23627758592, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110778113024, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110778113024, + "device": 3, + "inactive_split_bytes_current": 7435502592, + "inactive_split_bytes_peak": 9008247808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112963092480, + "reserved_bytes_peak": 112963092480, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T12:26:04.886517Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110778113024, + "allocated_bytes_peak_delta": 23627758592, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87150354432, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87150354432, + "device": 3, + "inactive_split_bytes_current": 6617613312, + "inactive_split_bytes_peak": 9008247808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 94474600448, + "reserved_bytes_peak": 94474600448, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:26:03.792894Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7435502592, + "inactive_split_bytes_current_delta": 817889280, + "inactive_split_bytes_peak": 9008247808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112963092480, + "reserved_bytes_current_delta": 18488492032, + "reserved_bytes_peak": 112963092480, + "reserved_bytes_peak_delta": 18488492032, + "segment_count_current": 591, + "segment_count_current_delta": 79, + "segment_count_peak": 591, + "segment_count_peak_delta": 79, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T122604886911Z.pickle", + "timestamp_utc": "2026-08-01T12:26:04.993613Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T122604886911Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T122604886911Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6cdc278038961d731d5afc69dc693153092e2b72 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T122604886911Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b20ae5cc55e77f76fd221f51cc497d95816109f2ea1e5753b786f926cd089fc +size 5582799 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T122605993431Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T122605993431Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e1fec76d810630a0f37a8ddb930d7929c77f0137 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T122605993431Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110780829696, + "active_bytes_peak_delta": 2716672, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110780829696, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110780829696, + "device": 3, + "inactive_split_bytes_current": 6686819328, + "inactive_split_bytes_peak": 9008247808, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112965189632, + "reserved_bytes_peak": 112965189632, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:05.993033Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110780829696, + "allocated_bytes_peak_delta": 2716672, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110778113024, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110778113024, + "device": 3, + "inactive_split_bytes_current": 7435502592, + "inactive_split_bytes_peak": 9008247808, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112963092480, + "reserved_bytes_peak": 112963092480, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T12:26:05.009017Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6686819328, + "inactive_split_bytes_current_delta": -748683264, + "inactive_split_bytes_peak": 9008247808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112965189632, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 112965189632, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 592, + "segment_count_current_delta": 1, + "segment_count_peak": 592, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T122605993431Z.pickle", + "timestamp_utc": "2026-08-01T12:26:06.097496Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T122605993431Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T122605993431Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..169d6f20a0f038e83910f07da65e5a008bd1628b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T122605993431Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb3e7864f7786dbebbb38ababe05589c29b1d5f5862807a485b734f1a364c575 +size 5585094 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T122608842389Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T122608842389Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a49ff39ed16f07fc4f26d43426c22405a7db2675 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T122608842389Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128942008320, + "active_bytes_peak_delta": 18161178624, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 14036464128, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:08.841806Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110789298176, + "allocated_bytes_peak_delta": 8468480, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110780829696, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110780829696, + "device": 3, + "inactive_split_bytes_current": 6686819328, + "inactive_split_bytes_peak": 9008247808, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112965189632, + "reserved_bytes_peak": 112965189632, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:06.112328Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_current_delta": 2910580736, + "inactive_split_bytes_peak": 14036464128, + "inactive_split_bytes_peak_delta": 5028216320, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142524547072, + "reserved_bytes_current_delta": 29559357440, + "reserved_bytes_peak": 142524547072, + "reserved_bytes_peak_delta": 29559357440, + "segment_count_current": 824, + "segment_count_current_delta": 232, + "segment_count_peak": 824, + "segment_count_peak_delta": 232, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T122608842389Z.pickle", + "timestamp_utc": "2026-08-01T12:26:08.962402Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T122608842389Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T122608842389Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b86e86154fad3ee263f1fcb9af70a620f759c651 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T122608842389Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7df6bb32212af37a3389e8dc74c4105e77825509b56a4e30806512a56da8e604 +size 5859024 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T122609901090Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T122609901090Z.json new file mode 100644 index 0000000000000000000000000000000000000000..df5b0a1b96147c35b634289505e6cd5cbc02bf1e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T122609901090Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128942008320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 14036464128, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:09.900618Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110789298176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 14036464128, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:08.978998Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14036464128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142524547072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142524547072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T122609901090Z.pickle", + "timestamp_utc": "2026-08-01T12:26:09.908276Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T122609901090Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T122609901090Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d7c7a66930ad1303f361d2c2ed970b4fd5701fd4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T122609901090Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c745a5f3668462d3ba733db0e64c8138ce0336d63ba10e7f350f3e4e83483486 +size 237264 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T122610830259Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T122610830259Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15331ea4305216425a40a3fee0bfe143029ddd94 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T122610830259Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128942008320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 14036464128, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:10.829850Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110789298176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 14036464128, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:09.909248Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14036464128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142524547072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142524547072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T122610830259Z.pickle", + "timestamp_utc": "2026-08-01T12:26:10.836935Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T122610830259Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T122610830259Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13fbab3461003102c3ae140eb53356b0f5a4fe99 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T122610830259Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a417bb52a1b25742510aab28d0c6dc0a4fb0a431e271301ef053bb81f4aee25 +size 237264 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T122611760911Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T122611760911Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b719545c2a1810d25ffcb70fd1f025a21425d673 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T122611760911Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128942008320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 14036464128, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:11.760523Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110789298176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 14036464128, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:10.837868Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14036464128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142524547072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142524547072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T122611760911Z.pickle", + "timestamp_utc": "2026-08-01T12:26:11.767174Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T122611760911Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T122611760911Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e999c030fbb67c8b876f702f4df0b83e705b7bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T122611760911Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5250c1f8c328fefdcb8236db8c88fbf36315ff0ba17b5e2cf6727eb495f4918d +size 237264 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T122612685684Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T122612685684Z.json new file mode 100644 index 0000000000000000000000000000000000000000..37339c74c1414d839772e6e9568a8b2bb2b5994b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T122612685684Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128942008320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 14036464128, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:12.685288Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110789298176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 14036464128, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:11.768088Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14036464128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142524547072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142524547072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T122612685684Z.pickle", + "timestamp_utc": "2026-08-01T12:26:12.692141Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T122612685684Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T122612685684Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..192a490014f003a544fac4ebcf90bff358a9d99f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T122612685684Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bb0e4055243c22e87f84f12ea5f70e0f16aae6613f9ab22d3746da642eca367 +size 237264 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T122613615206Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T122613615206Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d5b4faf84a7c9950538e9630f96b44f7fa0e4a97 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T122613615206Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128942008320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 14036464128, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:13.614805Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110789298176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 14036464128, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:12.693105Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14036464128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142524547072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142524547072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T122613615206Z.pickle", + "timestamp_utc": "2026-08-01T12:26:13.621566Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T122613615206Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T122613615206Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8d6a9d4da31ef57099e02ce4dc9fb38a13f41dee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T122613615206Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aee872d46ff5d58677bd132554be337e6e357b3b6b47758c65ab9155a3d6e9cf +size 237264 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T122714244904Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T122714244904Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eb296999a8ac3552575a3d14185c7bb70e26a188 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T122714244904Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 128942008320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 5446986752, + "inactive_split_bytes_peak": 14036464128, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 143417933824, + "reserved_bytes_peak": 143417933824, + "segment_count_current": 830, + "segment_count_peak": 830, + "timestamp_utc": "2026-08-01T12:27:14.244415Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110789298176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128942008320, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110789298176, + "device": 3, + "inactive_split_bytes_current": 9597400064, + "inactive_split_bytes_peak": 14036464128, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142524547072, + "reserved_bytes_peak": 142524547072, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:13.622472Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5446986752, + "inactive_split_bytes_current_delta": -4150413312, + "inactive_split_bytes_peak": 14036464128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 143417933824, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 143417933824, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 830, + "segment_count_current_delta": 6, + "segment_count_peak": 830, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T122714244904Z.pickle", + "timestamp_utc": "2026-08-01T12:27:14.335252Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T122714244904Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T122714244904Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e3d556672e8a632a64b8a71a71c9c81a28fa5117 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T122714244904Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e916e01d08774031de328d53df08aaa513be3fb826187c75580b4ecea814491c +size 4186549 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T122603660757Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T122603660757Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5c7e7f7f7e10247d2bf9ef79039679793bfe441 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T122603660757Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922216448, + "active_bytes_current_delta": 31690318848, + "active_bytes_peak": 87155494912, + "active_bytes_peak_delta": 51688716288, + "after": { + "active_bytes_current": 66922216448, + "active_bytes_peak": 87155494912, + "allocated_bytes_current": 66922216448, + "allocated_bytes_peak": 87155494912, + "device": 4, + "inactive_split_bytes_current": 6499075072, + "inactive_split_bytes_peak": 9783301120, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 94006935552, + "reserved_bytes_peak": 94006935552, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:26:03.660183Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922216448, + "allocated_bytes_current_delta": 31690318848, + "allocated_bytes_peak": 87155494912, + "allocated_bytes_peak_delta": 51688716288, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:24:54.433648Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6499075072, + "inactive_split_bytes_current_delta": 6366698496, + "inactive_split_bytes_peak": 9783301120, + "inactive_split_bytes_peak_delta": 9629891584, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 94006935552, + "reserved_bytes_current_delta": 54762930176, + "reserved_bytes_peak": 94006935552, + "reserved_bytes_peak_delta": 54762930176, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T122603660757Z.pickle", + "timestamp_utc": "2026-08-01T12:26:03.773255Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T122603660757Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T122603660757Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fd9e878877149bf59868e5ad0853090a93ba22fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T122603660757Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6546ccfb4c2074ed695bc5366ac7c14c68ee67e1de18895411b105c198b983e6 +size 5393525 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T122604886879Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T122604886879Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9dd80c27bb79a25e8a9d618ea0c11e92c42473d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T122604886879Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922216448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110785419776, + "active_bytes_peak_delta": 23629924864, + "after": { + "active_bytes_current": 66922216448, + "active_bytes_peak": 110785419776, + "allocated_bytes_current": 66922216448, + "allocated_bytes_peak": 110785419776, + "device": 4, + "inactive_split_bytes_current": 6908019712, + "inactive_split_bytes_peak": 9783301120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112730308608, + "reserved_bytes_peak": 112730308608, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:04.886475Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922216448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110785419776, + "allocated_bytes_peak_delta": 23629924864, + "before": { + "active_bytes_current": 66922216448, + "active_bytes_peak": 87155494912, + "allocated_bytes_current": 66922216448, + "allocated_bytes_peak": 87155494912, + "device": 4, + "inactive_split_bytes_current": 6499075072, + "inactive_split_bytes_peak": 9783301120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 94006935552, + "reserved_bytes_peak": 94006935552, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:26:03.791814Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6908019712, + "inactive_split_bytes_current_delta": 408944640, + "inactive_split_bytes_peak": 9783301120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112730308608, + "reserved_bytes_current_delta": 18723373056, + "reserved_bytes_peak": 112730308608, + "reserved_bytes_peak_delta": 18723373056, + "segment_count_current": 592, + "segment_count_current_delta": 80, + "segment_count_peak": 592, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T122604886879Z.pickle", + "timestamp_utc": "2026-08-01T12:26:04.993790Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T122604886879Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T122604886879Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..be4d5aad65a8d84067e8cdcf87370d8e12cc20c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T122604886879Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0886d4e5fffda124750e7c63d27d7582c9f8d341e293578ee643a27d110b314b +size 5586218 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T122605993416Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T122605993416Z.json new file mode 100644 index 0000000000000000000000000000000000000000..910dcb50b6bf38f5ce050f3c5878d94f1276f7bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T122605993416Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922216448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110790400000, + "active_bytes_peak_delta": 4980224, + "after": { + "active_bytes_current": 66922216448, + "active_bytes_peak": 110790400000, + "allocated_bytes_current": 66922216448, + "allocated_bytes_peak": 110790400000, + "device": 4, + "inactive_split_bytes_current": 6970934272, + "inactive_split_bytes_peak": 9783301120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112730308608, + "reserved_bytes_peak": 112730308608, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:05.993001Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922216448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110790400000, + "allocated_bytes_peak_delta": 4980224, + "before": { + "active_bytes_current": 66922216448, + "active_bytes_peak": 110785419776, + "allocated_bytes_current": 66922216448, + "allocated_bytes_peak": 110785419776, + "device": 4, + "inactive_split_bytes_current": 6908019712, + "inactive_split_bytes_peak": 9783301120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112730308608, + "reserved_bytes_peak": 112730308608, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:05.008927Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6970934272, + "inactive_split_bytes_current_delta": 62914560, + "inactive_split_bytes_peak": 9783301120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112730308608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112730308608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T122605993416Z.pickle", + "timestamp_utc": "2026-08-01T12:26:06.101914Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T122605993416Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T122605993416Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..272c9e2d14871695028d7b1055a3b3a173fb19cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T122605993416Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18c691ebe811c0e8cf2d1075bcfdc7726f3b300566cfe1046a2d79bd52e2c492 +size 5587686 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T122608842294Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T122608842294Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c743cab4e8882337990f702cb2b6a02512f5d766 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T122608842294Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939259904, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128950708224, + "active_bytes_peak_delta": 18160308224, + "after": { + "active_bytes_current": 66939259904, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66939259904, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_peak": 14010279936, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 142927200256, + "reserved_bytes_peak": 142927200256, + "segment_count_current": 817, + "segment_count_peak": 817, + "timestamp_utc": "2026-08-01T12:26:08.841762Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939259904, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110797998080, + "allocated_bytes_peak_delta": 7598080, + "before": { + "active_bytes_current": 66922216448, + "active_bytes_peak": 110790400000, + "allocated_bytes_current": 66922216448, + "allocated_bytes_peak": 110790400000, + "device": 4, + "inactive_split_bytes_current": 6970934272, + "inactive_split_bytes_peak": 9783301120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112730308608, + "reserved_bytes_peak": 112730308608, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:06.116810Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_current_delta": 3048992768, + "inactive_split_bytes_peak": 14010279936, + "inactive_split_bytes_peak_delta": 4226978816, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 142927200256, + "reserved_bytes_current_delta": 30196891648, + "reserved_bytes_peak": 142927200256, + "reserved_bytes_peak_delta": 30196891648, + "segment_count_current": 817, + "segment_count_current_delta": 225, + "segment_count_peak": 817, + "segment_count_peak_delta": 225, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T122608842294Z.pickle", + "timestamp_utc": "2026-08-01T12:26:08.964911Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T122608842294Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T122608842294Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..877012b1d0ac8331a09d05c5c9f918b324c04fca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T122608842294Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f51de2cae36e56c25200a335a5fba4e98da0c387b211c84624255b8c4a95419 +size 5925407 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T122609901033Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T122609901033Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc03264272a404734a2616dbc2fa695663fcf25e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T122609901033Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939259904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128950708224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939259904, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66939259904, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_peak": 14010279936, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 142927200256, + "reserved_bytes_peak": 142927200256, + "segment_count_current": 817, + "segment_count_peak": 817, + "timestamp_utc": "2026-08-01T12:26:09.900579Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939259904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797998080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939259904, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66939259904, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_peak": 14010279936, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 142927200256, + "reserved_bytes_peak": 142927200256, + "segment_count_current": 817, + "segment_count_peak": 817, + "timestamp_utc": "2026-08-01T12:26:08.981674Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14010279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 142927200256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142927200256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 817, + "segment_count_current_delta": 0, + "segment_count_peak": 817, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T122609901033Z.pickle", + "timestamp_utc": "2026-08-01T12:26:09.908345Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T122609901033Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T122609901033Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..373eb4204c2f71eb670e2ec9a70c00c4827f31b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T122609901033Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eec0713dccb3fdb4816af3609142be7b38d6beb6005e1052770d9e61864391a0 +size 241590 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T122610830315Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T122610830315Z.json new file mode 100644 index 0000000000000000000000000000000000000000..19d78df51ef458073fd9841fce2dc9442baaae23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T122610830315Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939259904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128950708224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939259904, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66939259904, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_peak": 14010279936, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 142927200256, + "reserved_bytes_peak": 142927200256, + "segment_count_current": 817, + "segment_count_peak": 817, + "timestamp_utc": "2026-08-01T12:26:10.829887Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939259904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797998080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939259904, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66939259904, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_peak": 14010279936, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 142927200256, + "reserved_bytes_peak": 142927200256, + "segment_count_current": 817, + "segment_count_peak": 817, + "timestamp_utc": "2026-08-01T12:26:09.909316Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14010279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 142927200256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142927200256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 817, + "segment_count_current_delta": 0, + "segment_count_peak": 817, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T122610830315Z.pickle", + "timestamp_utc": "2026-08-01T12:26:10.839133Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T122610830315Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T122610830315Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ad50ce6c702ac190d08a3756a7bd18ea4c5919bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T122610830315Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d3f39b7a03adfff17e2029ae04d9fdf9b9f1056d4e3c9abb34c6fcf643f732 +size 241590 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T122611760925Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T122611760925Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34ce4014607e9d579e61dcd02d368c89e10acded --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T122611760925Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939259904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128950708224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939259904, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66939259904, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_peak": 14010279936, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 142927200256, + "reserved_bytes_peak": 142927200256, + "segment_count_current": 817, + "segment_count_peak": 817, + "timestamp_utc": "2026-08-01T12:26:11.760534Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939259904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797998080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939259904, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66939259904, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_peak": 14010279936, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 142927200256, + "reserved_bytes_peak": 142927200256, + "segment_count_current": 817, + "segment_count_peak": 817, + "timestamp_utc": "2026-08-01T12:26:10.840096Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14010279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 142927200256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142927200256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 817, + "segment_count_current_delta": 0, + "segment_count_peak": 817, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T122611760925Z.pickle", + "timestamp_utc": "2026-08-01T12:26:11.767409Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T122611760925Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T122611760925Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e3ad6a4ce3b7fc4a1da279b761fb8fd6929b3173 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T122611760925Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fce99cc093ec389de414cbbc064df0886d0c590cd213422f54b26016b10ffcff +size 241590 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T122612685628Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T122612685628Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a82f3ba1c2ac15ffcdcf42116aee9cc28083e7f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T122612685628Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939259904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128950708224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939259904, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66939259904, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_peak": 14010279936, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 142927200256, + "reserved_bytes_peak": 142927200256, + "segment_count_current": 817, + "segment_count_peak": 817, + "timestamp_utc": "2026-08-01T12:26:12.685247Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939259904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797998080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939259904, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66939259904, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_peak": 14010279936, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 142927200256, + "reserved_bytes_peak": 142927200256, + "segment_count_current": 817, + "segment_count_peak": 817, + "timestamp_utc": "2026-08-01T12:26:11.768363Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14010279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 142927200256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142927200256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 817, + "segment_count_current_delta": 0, + "segment_count_peak": 817, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T122612685628Z.pickle", + "timestamp_utc": "2026-08-01T12:26:12.691900Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T122612685628Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T122612685628Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0b96c9949439d3ccdcb0443452e4dc5c4bae3472 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T122612685628Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78ad937ca3b880e0bb95c02e39e572b91d9ba0e9c76438b91bee2697aaafa3b9 +size 241590 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T122613615152Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T122613615152Z.json new file mode 100644 index 0000000000000000000000000000000000000000..659ce9cb19b8066b5fc7ec5686bf7f356a55b4c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T122613615152Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939259904, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128950708224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939259904, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66939259904, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_peak": 14010279936, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 142927200256, + "reserved_bytes_peak": 142927200256, + "segment_count_current": 817, + "segment_count_peak": 817, + "timestamp_utc": "2026-08-01T12:26:13.614768Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939259904, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110797998080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939259904, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66939259904, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_peak": 14010279936, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 142927200256, + "reserved_bytes_peak": 142927200256, + "segment_count_current": 817, + "segment_count_peak": 817, + "timestamp_utc": "2026-08-01T12:26:12.692835Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14010279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 142927200256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142927200256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 817, + "segment_count_current_delta": 0, + "segment_count_peak": 817, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T122613615152Z.pickle", + "timestamp_utc": "2026-08-01T12:26:13.621731Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T122613615152Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T122613615152Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5aee271be59cdc034484c0fcbe39b83cf191571f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T122613615152Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:788af6757b83f66affd53e193ea21302c42ddf9338654858f41030ba614b56d6 +size 241590 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T122714244906Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T122714244906Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27577099eadd122888dfea0779e2375ab01c179d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T122714244906Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66956186624, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 128950708224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66956186624, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66956186624, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 4642679808, + "inactive_split_bytes_peak": 14010279936, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 142996406272, + "reserved_bytes_peak": 142996406272, + "segment_count_current": 822, + "segment_count_peak": 822, + "timestamp_utc": "2026-08-01T12:27:14.244426Z", + "world_size": 8 + }, + "allocated_bytes_current": 66956186624, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110797998080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939259904, + "active_bytes_peak": 128950708224, + "allocated_bytes_current": 66939259904, + "allocated_bytes_peak": 110797998080, + "device": 4, + "inactive_split_bytes_current": 10019927040, + "inactive_split_bytes_peak": 14010279936, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 142927200256, + "reserved_bytes_peak": 142927200256, + "segment_count_current": 817, + "segment_count_peak": 817, + "timestamp_utc": "2026-08-01T12:26:13.622699Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4642679808, + "inactive_split_bytes_current_delta": -5377247232, + "inactive_split_bytes_peak": 14010279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 142996406272, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 142996406272, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 822, + "segment_count_current_delta": 5, + "segment_count_peak": 822, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T122714244906Z.pickle", + "timestamp_utc": "2026-08-01T12:27:14.333445Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T122714244906Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T122714244906Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d01a436ab6e2523afd01a87c23d314e48b6889db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T122714244906Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdfd5d88c74a2219a4f028a8cc95fe0fd3a111a3fd4660697fb470f9f989e855 +size 4351903 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T122603660789Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T122603660789Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d6adbdfe908a1d85abf2810b86d52fd54db0af2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T122603660789Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 87150354432, + "active_bytes_peak_delta": 51684018176, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87150354432, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87150354432, + "device": 5, + "inactive_split_bytes_current": 6617613312, + "inactive_split_bytes_peak": 9125688320, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 94474600448, + "reserved_bytes_peak": 94474600448, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:26:03.660198Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 87150354432, + "allocated_bytes_peak_delta": 51684018176, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:24:54.434301Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6617613312, + "inactive_split_bytes_current_delta": 6484794368, + "inactive_split_bytes_peak": 9125688320, + "inactive_split_bytes_peak_delta": 8965302272, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 94474600448, + "reserved_bytes_current_delta": 55230595072, + "reserved_bytes_peak": 94474600448, + "reserved_bytes_peak_delta": 55230595072, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T122603660789Z.pickle", + "timestamp_utc": "2026-08-01T12:26:03.772079Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T122603660789Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T122603660789Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dd223721d2a40fa04023c24f367bc884238debf9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T122603660789Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f904e7a7a9b3580abe3fec604e66948f4a92d2f931e9f57305019e2f993d8f6a +size 5388939 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T122604886912Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T122604886912Z.json new file mode 100644 index 0000000000000000000000000000000000000000..972833615eba4fe96af4329f81cca89f547afa7e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T122604886912Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110773655552, + "active_bytes_peak_delta": 23623301120, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110773655552, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110773655552, + "device": 5, + "inactive_split_bytes_current": 7332742144, + "inactive_split_bytes_peak": 9158510592, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113080532992, + "reserved_bytes_peak": 113080532992, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:04.886494Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110773655552, + "allocated_bytes_peak_delta": 23623301120, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87150354432, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87150354432, + "device": 5, + "inactive_split_bytes_current": 6617613312, + "inactive_split_bytes_peak": 9125688320, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 94474600448, + "reserved_bytes_peak": 94474600448, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:26:03.791620Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7332742144, + "inactive_split_bytes_current_delta": 715128832, + "inactive_split_bytes_peak": 9158510592, + "inactive_split_bytes_peak_delta": 32822272, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113080532992, + "reserved_bytes_current_delta": 18605932544, + "reserved_bytes_peak": 113080532992, + "reserved_bytes_peak_delta": 18605932544, + "segment_count_current": 592, + "segment_count_current_delta": 80, + "segment_count_peak": 592, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T122604886912Z.pickle", + "timestamp_utc": "2026-08-01T12:26:04.993206Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T122604886912Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T122604886912Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4d66db191c35f6b23e7de67a98b86e7e25c25751 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T122604886912Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db8d99907615433bd85d7027d950c462b659234734aae2ad0e5cd15d21a9083c +size 5584989 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T122605993426Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T122605993426Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f7aac39618f6a9931f8ae9271c5191ee8c0bdf4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T122605993426Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110778208256, + "active_bytes_peak_delta": 4552704, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110778208256, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110778208256, + "device": 5, + "inactive_split_bytes_current": 6854591488, + "inactive_split_bytes_peak": 9584048128, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113080532992, + "reserved_bytes_peak": 113080532992, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:05.993016Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110778208256, + "allocated_bytes_peak_delta": 4552704, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110773655552, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110773655552, + "device": 5, + "inactive_split_bytes_current": 7332742144, + "inactive_split_bytes_peak": 9158510592, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113080532992, + "reserved_bytes_peak": 113080532992, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:05.008456Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6854591488, + "inactive_split_bytes_current_delta": -478150656, + "inactive_split_bytes_peak": 9584048128, + "inactive_split_bytes_peak_delta": 425537536, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113080532992, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113080532992, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T122605993426Z.pickle", + "timestamp_utc": "2026-08-01T12:26:06.097025Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T122605993426Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T122605993426Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b62b12955d1b64c6f1a6dc19b3538d4b1e7a071c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T122605993426Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:438cff2c97316b325df21856ff6825b2b12f03dc85752b878b1fd65bb606a058 +size 5585721 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T122608842427Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T122608842427Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1a1348f036bcd226b4682b8f1c32d7a96e72239 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T122608842427Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 18165897216, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14531414016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142633598976, + "reserved_bytes_peak": 142633598976, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:08.841839Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 13187072, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110778208256, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110778208256, + "device": 5, + "inactive_split_bytes_current": 6854591488, + "inactive_split_bytes_peak": 9584048128, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113080532992, + "reserved_bytes_peak": 113080532992, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T12:26:06.111374Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_current_delta": 3319525376, + "inactive_split_bytes_peak": 14531414016, + "inactive_split_bytes_peak_delta": 4947365888, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142633598976, + "reserved_bytes_current_delta": 29553065984, + "reserved_bytes_peak": 142633598976, + "reserved_bytes_peak_delta": 29553065984, + "segment_count_current": 824, + "segment_count_current_delta": 232, + "segment_count_peak": 824, + "segment_count_peak_delta": 232, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T122608842427Z.pickle", + "timestamp_utc": "2026-08-01T12:26:08.960989Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T122608842427Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T122608842427Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..627063458018a475eea853cad23e555721da570e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T122608842427Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6396daf0c7b9d2d72e111f4a13567f14b9c17e9a3ec23f249fc1693f0fa55270 +size 5859872 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T122609901072Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T122609901072Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3c390ff25e2f956f681925b3024e8373905405a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T122609901072Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14531414016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142633598976, + "reserved_bytes_peak": 142633598976, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:09.900608Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14531414016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142633598976, + "reserved_bytes_peak": 142633598976, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:08.977704Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14531414016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142633598976, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142633598976, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T122609901072Z.pickle", + "timestamp_utc": "2026-08-01T12:26:09.908478Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T122609901072Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T122609901072Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3c65ac1eda579c5d03bd5d7398f5a1f4ed0c1a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T122609901072Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4537f8f5684d7677d2e2362a02612d072eb5e216b9846f5ce4212b9ed658377 +size 236618 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T122610830286Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T122610830286Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a6c708d070bd0c536d5d45a6ef26bc534343dc9e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T122610830286Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14531414016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142633598976, + "reserved_bytes_peak": 142633598976, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:10.829861Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14531414016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142633598976, + "reserved_bytes_peak": 142633598976, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:09.909482Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14531414016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142633598976, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142633598976, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T122610830286Z.pickle", + "timestamp_utc": "2026-08-01T12:26:10.836999Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T122610830286Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T122610830286Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0c9dd75acc2462439bd5cd8cf839b665c46c4e60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T122610830286Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4c86dda44adff170fb5541f6ef688707360aa1828506e02520d8efeb1657b4e +size 236618 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T122611760885Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T122611760885Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f49eef358cf5d908d984738542699b1830759db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T122611760885Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14531414016, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142633598976, + "reserved_bytes_peak": 142633598976, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:11.760488Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14531414016, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142633598976, + "reserved_bytes_peak": 142633598976, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:10.837955Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14531414016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142633598976, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142633598976, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T122611760885Z.pickle", + "timestamp_utc": "2026-08-01T12:26:11.767186Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T122611760885Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T122611760885Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..42f81266c741f31db534fd8b8538e46649e247fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T122611760885Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:201c0c48779edc14e640de48429e6d13ebf4b661d92867f5c58a4b663c482a5e +size 236587 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T122612685716Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T122612685716Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9ae41a72a6824c016cd32c1c4241952567d681a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T122612685716Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 10172019712, + "inactive_split_bytes_peak": 14531414016, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142633598976, + "reserved_bytes_peak": 142633598976, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:12.685322Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14531414016, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142633598976, + "reserved_bytes_peak": 142633598976, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:11.768138Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10172019712, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 14531414016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142633598976, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142633598976, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T122612685716Z.pickle", + "timestamp_utc": "2026-08-01T12:26:12.692540Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T122612685716Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T122612685716Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..acf7919653481c7237f8132db467e027950d146f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T122612685716Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af36917653fbf6254402f17fba69e27a0f790c1c6b8a73605ec7c42a1c5f1269 +size 236584 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T122613615174Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T122613615174Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4bf719b2f3857c93fc58f9714276e7eb8e92d5da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T122613615174Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 10172019712, + "inactive_split_bytes_peak": 14531414016, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142633598976, + "reserved_bytes_peak": 142633598976, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:13.614779Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 10172019712, + "inactive_split_bytes_peak": 14531414016, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142633598976, + "reserved_bytes_peak": 142633598976, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:12.693504Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10172019712, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14531414016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142633598976, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142633598976, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 824, + "segment_count_current_delta": 0, + "segment_count_peak": 824, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T122613615174Z.pickle", + "timestamp_utc": "2026-08-01T12:26:13.621588Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T122613615174Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T122613615174Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ff04b85fca86670fe33ae5652bdbf7564f6eb41 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T122613615174Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95aac4e7ecece4789abec316bf576b30fd8dffa61fbd74e241a0d2cb768b71a6 +size 236615 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T122714244904Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T122714244904Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d0952165749145766161533bb511a9f7b5db704 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T122714244904Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 5151288320, + "inactive_split_bytes_peak": 14531414016, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 143526985728, + "reserved_bytes_peak": 143526985728, + "segment_count_current": 830, + "segment_count_peak": 830, + "timestamp_utc": "2026-08-01T12:27:14.244425Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 5, + "inactive_split_bytes_current": 10172019712, + "inactive_split_bytes_peak": 14531414016, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142633598976, + "reserved_bytes_peak": 142633598976, + "segment_count_current": 824, + "segment_count_peak": 824, + "timestamp_utc": "2026-08-01T12:26:13.622537Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5151288320, + "inactive_split_bytes_current_delta": -5020731392, + "inactive_split_bytes_peak": 14531414016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 143526985728, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 143526985728, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 830, + "segment_count_current_delta": 6, + "segment_count_peak": 830, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T122714244904Z.pickle", + "timestamp_utc": "2026-08-01T12:27:14.333525Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T122714244904Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T122714244904Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8126e4bff5cc2e427866ab01862aba16eb8c0f1e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T122714244904Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f04f83329af610cb120c5ddad07fd1652f109b47123e0cf7658b4b81698c37a6 +size 4185688 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T122603660829Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T122603660829Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c4052530304ec7f9f0d30abadc8480c26925bc4e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T122603660829Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 87150354432, + "active_bytes_peak_delta": 51684018176, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87150354432, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87150354432, + "device": 6, + "inactive_split_bytes_current": 6611321856, + "inactive_split_bytes_peak": 9117299712, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 94476697600, + "reserved_bytes_peak": 94476697600, + "segment_count_current": 513, + "segment_count_peak": 513, + "timestamp_utc": "2026-08-01T12:26:03.660248Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 87150354432, + "allocated_bytes_peak_delta": 51684018176, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:24:54.433964Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6611321856, + "inactive_split_bytes_current_delta": 6478502912, + "inactive_split_bytes_peak": 9117299712, + "inactive_split_bytes_peak_delta": 8956913664, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 94476697600, + "reserved_bytes_current_delta": 55232692224, + "reserved_bytes_peak": 94476697600, + "reserved_bytes_peak_delta": 55232692224, + "segment_count_current": 513, + "segment_count_current_delta": 442, + "segment_count_peak": 513, + "segment_count_peak_delta": 442, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T122603660829Z.pickle", + "timestamp_utc": "2026-08-01T12:26:03.771889Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T122603660829Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T122603660829Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d1029005e5e10f37d21de1986ef4a7f4f9494883 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T122603660829Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95a9c1af8853d8eb725549be8566a87f0a82244010ce7c18fe5e78675f3650f8 +size 5388359 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T122604886927Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T122604886927Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b86d00c63bbdf88102afd2076d05bce06a11fdaf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T122604886927Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110773655552, + "active_bytes_peak_delta": 23623301120, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110773655552, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110773655552, + "device": 6, + "inactive_split_bytes_current": 7328547840, + "inactive_split_bytes_peak": 9156413440, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113082630144, + "reserved_bytes_peak": 113082630144, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T12:26:04.886518Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110773655552, + "allocated_bytes_peak_delta": 23623301120, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87150354432, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87150354432, + "device": 6, + "inactive_split_bytes_current": 6611321856, + "inactive_split_bytes_peak": 9117299712, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 94476697600, + "reserved_bytes_peak": 94476697600, + "segment_count_current": 513, + "segment_count_peak": 513, + "timestamp_utc": "2026-08-01T12:26:03.789346Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7328547840, + "inactive_split_bytes_current_delta": 717225984, + "inactive_split_bytes_peak": 9156413440, + "inactive_split_bytes_peak_delta": 39113728, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113082630144, + "reserved_bytes_current_delta": 18605932544, + "reserved_bytes_peak": 113082630144, + "reserved_bytes_peak_delta": 18605932544, + "segment_count_current": 593, + "segment_count_current_delta": 80, + "segment_count_peak": 593, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T122604886927Z.pickle", + "timestamp_utc": "2026-08-01T12:26:04.990720Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T122604886927Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T122604886927Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6d52f3cdf49ee1d4b3413dfef9a53f3d898e0873 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T122604886927Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cee330bb0d478ec9374d16ca8850707444ec4f15dd5b0b8001de9593aee12dc8 +size 5583917 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T122605993417Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T122605993417Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e3d24e30b90b9782aabe46d00242a7a020f2b8a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T122605993417Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110778208256, + "active_bytes_peak_delta": 4552704, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110778208256, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110778208256, + "device": 6, + "inactive_split_bytes_current": 6858785792, + "inactive_split_bytes_peak": 9690397696, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113082630144, + "reserved_bytes_peak": 113082630144, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T12:26:05.993014Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110778208256, + "allocated_bytes_peak_delta": 4552704, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110773655552, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110773655552, + "device": 6, + "inactive_split_bytes_current": 7328547840, + "inactive_split_bytes_peak": 9156413440, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113082630144, + "reserved_bytes_peak": 113082630144, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T12:26:05.004030Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6858785792, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 9690397696, + "inactive_split_bytes_peak_delta": 533984256, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113082630144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113082630144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 593, + "segment_count_current_delta": 0, + "segment_count_peak": 593, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T122605993417Z.pickle", + "timestamp_utc": "2026-08-01T12:26:06.092557Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T122605993417Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T122605993417Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2fc89751fb6cb30082ea780fcb18298c96af022b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T122605993417Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6b6dcecdb155956b6b7ffa0364a5d8f96c9a1e5be889c014dc4d328ce5a37a4 +size 5586244 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T122608842471Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T122608842471Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a6100c819a95419aad067b0b58520887f9a3944a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T122608842471Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 18165897216, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14529316864, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142637793280, + "reserved_bytes_peak": 142637793280, + "segment_count_current": 826, + "segment_count_peak": 826, + "timestamp_utc": "2026-08-01T12:26:08.841892Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 13187072, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110778208256, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110778208256, + "device": 6, + "inactive_split_bytes_current": 6858785792, + "inactive_split_bytes_peak": 9690397696, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113082630144, + "reserved_bytes_peak": 113082630144, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T12:26:06.105139Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_current_delta": 3315331072, + "inactive_split_bytes_peak": 14529316864, + "inactive_split_bytes_peak_delta": 4838919168, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142637793280, + "reserved_bytes_current_delta": 29555163136, + "reserved_bytes_peak": 142637793280, + "reserved_bytes_peak_delta": 29555163136, + "segment_count_current": 826, + "segment_count_current_delta": 233, + "segment_count_peak": 826, + "segment_count_peak_delta": 233, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T122608842471Z.pickle", + "timestamp_utc": "2026-08-01T12:26:08.959326Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T122608842471Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T122608842471Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c3a246de0042bdbd2262f759aec31fbf7f16ef7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T122608842471Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c5daf91c1a602dfb42e31bf8cf7f4be4942cfe33db195595a35c1ea9b1d1cd0 +size 5858119 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T122609901088Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T122609901088Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ee7b1ca5e93f557cc456ae71ef379f41d90a8e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T122609901088Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14529316864, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142637793280, + "reserved_bytes_peak": 142637793280, + "segment_count_current": 826, + "segment_count_peak": 826, + "timestamp_utc": "2026-08-01T12:26:09.900606Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14529316864, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142637793280, + "reserved_bytes_peak": 142637793280, + "segment_count_current": 826, + "segment_count_peak": 826, + "timestamp_utc": "2026-08-01T12:26:08.973675Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14529316864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142637793280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142637793280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 826, + "segment_count_current_delta": 0, + "segment_count_peak": 826, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T122609901088Z.pickle", + "timestamp_utc": "2026-08-01T12:26:09.908509Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T122609901088Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T122609901088Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..10b28f86d355cccab3ec7de92416f1238487c2c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T122609901088Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c9cd46aab461556d5c60fb94128707e4cb18acac23aa7ae5295cfeb4f906e6b +size 234945 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T122610830301Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T122610830301Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31b10acf03703251bcae665a453b1c664dab2e25 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T122610830301Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14529316864, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142637793280, + "reserved_bytes_peak": 142637793280, + "segment_count_current": 826, + "segment_count_peak": 826, + "timestamp_utc": "2026-08-01T12:26:10.829872Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14529316864, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142637793280, + "reserved_bytes_peak": 142637793280, + "segment_count_current": 826, + "segment_count_peak": 826, + "timestamp_utc": "2026-08-01T12:26:09.909495Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14529316864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142637793280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142637793280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 826, + "segment_count_current_delta": 0, + "segment_count_peak": 826, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T122610830301Z.pickle", + "timestamp_utc": "2026-08-01T12:26:10.837229Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T122610830301Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T122610830301Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03768b8aafe8f4680ecafb876f9ee43ff199735c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T122610830301Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0399a5816bb91bf81252e08fef179367da9837cf67b8ef2b9ede9c32a2a1584f +size 234945 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T122611760990Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T122611760990Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c4feafd8f10ce63d05ff234d4987b0ca5175d84f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T122611760990Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14529316864, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142637793280, + "reserved_bytes_peak": 142637793280, + "segment_count_current": 826, + "segment_count_peak": 826, + "timestamp_utc": "2026-08-01T12:26:11.760553Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14529316864, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142637793280, + "reserved_bytes_peak": 142637793280, + "segment_count_current": 826, + "segment_count_peak": 826, + "timestamp_utc": "2026-08-01T12:26:10.838205Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14529316864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142637793280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142637793280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 826, + "segment_count_current_delta": 0, + "segment_count_peak": 826, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T122611760990Z.pickle", + "timestamp_utc": "2026-08-01T12:26:11.767612Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T122611760990Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T122611760990Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ad5bcb5940118a2030ef3789e924492cd8a3f5f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T122611760990Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02db57dd93ea79692ffbe45a403caa21edfe6469e24c985e65779217209f19fe +size 234945 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T122612685700Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T122612685700Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d0ed9fcd646a13afae563b05577978ae46b6fa32 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T122612685700Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14529316864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142637793280, + "reserved_bytes_peak": 142637793280, + "segment_count_current": 826, + "segment_count_peak": 826, + "timestamp_utc": "2026-08-01T12:26:12.685289Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14529316864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142637793280, + "reserved_bytes_peak": 142637793280, + "segment_count_current": 826, + "segment_count_peak": 826, + "timestamp_utc": "2026-08-01T12:26:11.768584Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14529316864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142637793280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142637793280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 826, + "segment_count_current_delta": 0, + "segment_count_peak": 826, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T122612685700Z.pickle", + "timestamp_utc": "2026-08-01T12:26:12.692131Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T122612685700Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T122612685700Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..39a1659c54865ecbb9f92f383783c52b737e4961 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T122612685700Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33c63897317720115272349be5c3f7b5c7e3c1bf88d71962e82cb2a588be65d9 +size 234945 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T122613615215Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T122613615215Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3703f0db992485de4f93934cc6908ee5646c6053 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T122613615215Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14529316864, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142637793280, + "reserved_bytes_peak": 142637793280, + "segment_count_current": 826, + "segment_count_peak": 826, + "timestamp_utc": "2026-08-01T12:26:13.614807Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14529316864, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142637793280, + "reserved_bytes_peak": 142637793280, + "segment_count_current": 826, + "segment_count_peak": 826, + "timestamp_utc": "2026-08-01T12:26:12.693074Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14529316864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142637793280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142637793280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 826, + "segment_count_current_delta": 0, + "segment_count_peak": 826, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T122613615215Z.pickle", + "timestamp_utc": "2026-08-01T12:26:13.622075Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T122613615215Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T122613615215Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..96b87499f8e1b28927b35f92fd970e953b9047bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T122613615215Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08a83318c1a6bfab4824c2fb7122122da44f36e5c742718ac99513b6d0e59e9d +size 234945 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T122714244907Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T122714244907Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef51b80d7d599643322b42d3716915c7baf4b0cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T122714244907Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 128944105472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 5147094016, + "inactive_split_bytes_peak": 14529316864, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 143531180032, + "reserved_bytes_peak": 143531180032, + "segment_count_current": 832, + "segment_count_peak": 832, + "timestamp_utc": "2026-08-01T12:27:14.244433Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110791395328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 128944105472, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110791395328, + "device": 6, + "inactive_split_bytes_current": 10174116864, + "inactive_split_bytes_peak": 14529316864, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142637793280, + "reserved_bytes_peak": 142637793280, + "segment_count_current": 826, + "segment_count_peak": 826, + "timestamp_utc": "2026-08-01T12:26:13.623018Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5147094016, + "inactive_split_bytes_current_delta": -5027022848, + "inactive_split_bytes_peak": 14529316864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 143531180032, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 143531180032, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 832, + "segment_count_current_delta": 6, + "segment_count_peak": 832, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T122714244907Z.pickle", + "timestamp_utc": "2026-08-01T12:27:14.336209Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T122714244907Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T122714244907Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..61a82ce48dffdcc4df950f87f4ede0a3b1027154 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T122714244907Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52d351cc9de06ddb95e9aae7143701c5f6bb99c157b8f41b5fbed36316313863 +size 4185882 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T122603660700Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T122603660700Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bde7c98d40dd924c388455a6945a7e18a3fb68b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T122603660700Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 31689664000, + "active_bytes_peak": 87150354432, + "active_bytes_peak_delta": 51684018176, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 87150354432, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 87150354432, + "device": 7, + "inactive_split_bytes_current": 6617612800, + "inactive_split_bytes_peak": 9008247808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 94474600448, + "reserved_bytes_peak": 94474600448, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:26:03.660211Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 31689664000, + "allocated_bytes_peak": 87150354432, + "allocated_bytes_peak_delta": 51684018176, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:24:54.433672Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6617612800, + "inactive_split_bytes_current_delta": 6484793856, + "inactive_split_bytes_peak": 9008247808, + "inactive_split_bytes_peak_delta": 8847861760, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 94474600448, + "reserved_bytes_current_delta": 55230595072, + "reserved_bytes_peak": 94474600448, + "reserved_bytes_peak_delta": 55230595072, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T122603660700Z.pickle", + "timestamp_utc": "2026-08-01T12:26:03.772426Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T122603660700Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T122603660700Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6f8d0592feb485b00698d79cdfc887d94f344d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T122603660700Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c078e42a6a070bb7c8ba5618db80bbe11b6af6e2927d48cabfddc4d54dd89fba +size 5390593 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T122604887007Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T122604887007Z.json new file mode 100644 index 0000000000000000000000000000000000000000..227f7ea9b8be69dce5ab7b39ac98863eb8555408 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T122604887007Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110773656064, + "active_bytes_peak_delta": 23623301632, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 110773656064, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 110773656064, + "device": 7, + "inactive_split_bytes_current": 7328547328, + "inactive_split_bytes_peak": 9156412928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113082630144, + "reserved_bytes_peak": 113082630144, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T12:26:04.886575Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110773656064, + "allocated_bytes_peak_delta": 23623301632, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 87150354432, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 87150354432, + "device": 7, + "inactive_split_bytes_current": 6617612800, + "inactive_split_bytes_peak": 9008247808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 94474600448, + "reserved_bytes_peak": 94474600448, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T12:26:03.792456Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7328547328, + "inactive_split_bytes_current_delta": 710934528, + "inactive_split_bytes_peak": 9156412928, + "inactive_split_bytes_peak_delta": 148165120, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113082630144, + "reserved_bytes_current_delta": 18608029696, + "reserved_bytes_peak": 113082630144, + "reserved_bytes_peak_delta": 18608029696, + "segment_count_current": 593, + "segment_count_current_delta": 81, + "segment_count_peak": 593, + "segment_count_peak_delta": 81, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T122604887007Z.pickle", + "timestamp_utc": "2026-08-01T12:26:04.994508Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T122604887007Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T122604887007Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4cfbcb57eeae98ab7557e4500bf92fe0ba1d0c03 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T122604887007Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:975407d14bd06ecab6659eec62f3a6777fd54341ecd15cf1327d2f0ee26609d0 +size 5586196 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T122605993480Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T122605993480Z.json new file mode 100644 index 0000000000000000000000000000000000000000..250dcc2f58034c182b4126afde7aa2c3755710fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T122605993480Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110778208768, + "active_bytes_peak_delta": 4552704, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 110778208768, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 110778208768, + "device": 7, + "inactive_split_bytes_current": 6852493824, + "inactive_split_bytes_peak": 9624122880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113082630144, + "reserved_bytes_peak": 113082630144, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T12:26:05.993065Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110778208768, + "allocated_bytes_peak_delta": 4552704, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 110773656064, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 110773656064, + "device": 7, + "inactive_split_bytes_current": 7328547328, + "inactive_split_bytes_peak": 9156412928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113082630144, + "reserved_bytes_peak": 113082630144, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T12:26:05.010101Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6852493824, + "inactive_split_bytes_current_delta": -476053504, + "inactive_split_bytes_peak": 9624122880, + "inactive_split_bytes_peak_delta": 467709952, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113082630144, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113082630144, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 593, + "segment_count_current_delta": 0, + "segment_count_peak": 593, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T122605993480Z.pickle", + "timestamp_utc": "2026-08-01T12:26:06.098405Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T122605993480Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T122605993480Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5b5b0c745e9af8022de42ac06b163174955c495 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T122605993480Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdbb003b57f2c83840492be9242f568c68c41f59a7084336632d7356c9b3588a +size 5587378 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T122608842414Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T122608842414Z.json new file mode 100644 index 0000000000000000000000000000000000000000..58b11b7269992440360bcc3f05e35f0884838728 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T122608842414Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128946465280, + "active_bytes_peak_delta": 18168256512, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_peak": 14221501952, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142883160064, + "reserved_bytes_peak": 142883160064, + "segment_count_current": 834, + "segment_count_peak": 834, + "timestamp_utc": "2026-08-01T12:26:08.841844Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110792706560, + "allocated_bytes_peak_delta": 14497792, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 110778208768, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 110778208768, + "device": 7, + "inactive_split_bytes_current": 6852493824, + "inactive_split_bytes_peak": 9624122880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113082630144, + "reserved_bytes_peak": 113082630144, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T12:26:06.113401Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_current_delta": 3904630784, + "inactive_split_bytes_peak": 14221501952, + "inactive_split_bytes_peak_delta": 4597379072, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142883160064, + "reserved_bytes_current_delta": 29800529920, + "reserved_bytes_peak": 142883160064, + "reserved_bytes_peak_delta": 29800529920, + "segment_count_current": 834, + "segment_count_current_delta": 241, + "segment_count_peak": 834, + "segment_count_peak_delta": 241, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T122608842414Z.pickle", + "timestamp_utc": "2026-08-01T12:26:08.962706Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T122608842414Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T122608842414Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1cfdbd3a08c63109e189516b129a78ef56c470b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T122608842414Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddc1f7b826525ef7ce0ddc1339f5f64d5cd901f17e44547ee0f17a0f8e0bd24d +size 5863185 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T122609901128Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T122609901128Z.json new file mode 100644 index 0000000000000000000000000000000000000000..74f4f78324671c9a1afe26bf7287d5996c567b2b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T122609901128Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128946465280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_peak": 14221501952, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142883160064, + "reserved_bytes_peak": 142883160064, + "segment_count_current": 834, + "segment_count_peak": 834, + "timestamp_utc": "2026-08-01T12:26:09.900645Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110792706560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_peak": 14221501952, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142883160064, + "reserved_bytes_peak": 142883160064, + "segment_count_current": 834, + "segment_count_peak": 834, + "timestamp_utc": "2026-08-01T12:26:08.979423Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14221501952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142883160064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142883160064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 834, + "segment_count_current_delta": 0, + "segment_count_peak": 834, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T122609901128Z.pickle", + "timestamp_utc": "2026-08-01T12:26:09.908251Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T122609901128Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T122609901128Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5647475f0001d27b63b4f6cc5dbb6c5c1cca99c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T122609901128Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d70951f9b4c6a473e9772190ce5a7c797cd58ef797726c2845634f351101e504 +size 238361 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T122610830411Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T122610830411Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8194fc60207c5f898ecf75913634528e6388374e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T122610830411Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128946465280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_peak": 14221501952, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142883160064, + "reserved_bytes_peak": 142883160064, + "segment_count_current": 834, + "segment_count_peak": 834, + "timestamp_utc": "2026-08-01T12:26:10.829964Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110792706560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_peak": 14221501952, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142883160064, + "reserved_bytes_peak": 142883160064, + "segment_count_current": 834, + "segment_count_peak": 834, + "timestamp_utc": "2026-08-01T12:26:09.909236Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14221501952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142883160064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142883160064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 834, + "segment_count_current_delta": 0, + "segment_count_peak": 834, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T122610830411Z.pickle", + "timestamp_utc": "2026-08-01T12:26:10.838717Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T122610830411Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T122610830411Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..618307cefc6c58ce110abb84ab662d8f3c165d0f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T122610830411Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe922dbe2ffada9a549e9fb8d5c0c814ed8cbd3f4d3a4644447113ff7017d8b5 +size 238361 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T122611760957Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T122611760957Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a5b5a643fc52719687512f46f970d9749c7b3571 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T122611760957Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128946465280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_peak": 14221501952, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142883160064, + "reserved_bytes_peak": 142883160064, + "segment_count_current": 834, + "segment_count_peak": 834, + "timestamp_utc": "2026-08-01T12:26:11.760564Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110792706560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_peak": 14221501952, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142883160064, + "reserved_bytes_peak": 142883160064, + "segment_count_current": 834, + "segment_count_peak": 834, + "timestamp_utc": "2026-08-01T12:26:10.839709Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14221501952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142883160064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142883160064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 834, + "segment_count_current_delta": 0, + "segment_count_peak": 834, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T122611760957Z.pickle", + "timestamp_utc": "2026-08-01T12:26:11.767309Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T122611760957Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T122611760957Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ad52d942e39c1a50777a60dd29996bbf291e4cf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T122611760957Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c92bcc977f66b90edb9b442dc65eb72223c550bb975ed47887fa3dda6c609ec +size 238362 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T122612685718Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T122612685718Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fac276bdff588a34826172efec00b4b7a0c51861 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T122612685718Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128946465280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_peak": 14221501952, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142883160064, + "reserved_bytes_peak": 142883160064, + "segment_count_current": 834, + "segment_count_peak": 834, + "timestamp_utc": "2026-08-01T12:26:12.685315Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110792706560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_peak": 14221501952, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142883160064, + "reserved_bytes_peak": 142883160064, + "segment_count_current": 834, + "segment_count_peak": 834, + "timestamp_utc": "2026-08-01T12:26:11.768269Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14221501952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142883160064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142883160064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 834, + "segment_count_current_delta": 0, + "segment_count_peak": 834, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T122612685718Z.pickle", + "timestamp_utc": "2026-08-01T12:26:12.691974Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T122612685718Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T122612685718Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df13506c8e75446b6f0694bd7378b99d9309fb34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T122612685718Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3566209c42c709618453265faa4caa229708e8d18adee55f0b12914279b93412 +size 238361 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T122613615230Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T122613615230Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f391b2591275b423ff28e7d061d86c01d69499a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T122613615230Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128946465280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_peak": 14221501952, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142883160064, + "reserved_bytes_peak": 142883160064, + "segment_count_current": 834, + "segment_count_peak": 834, + "timestamp_utc": "2026-08-01T12:26:13.614824Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110792706560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_peak": 14221501952, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142883160064, + "reserved_bytes_peak": 142883160064, + "segment_count_current": 834, + "segment_count_peak": 834, + "timestamp_utc": "2026-08-01T12:26:12.692944Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14221501952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142883160064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142883160064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 834, + "segment_count_current_delta": 0, + "segment_count_peak": 834, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T122613615230Z.pickle", + "timestamp_utc": "2026-08-01T12:26:13.622131Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T122613615230Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T122613615230Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..224087bd7a1132d7a368bc8cd3361954c82c0926 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T122613615230Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30645a21b7d95be944c49ca74489f214572a694c2bf3c96b0e5639939cef9cde +size 238333 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T122714244914Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T122714244914Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7b6668bbcf86945b1893f251ea62f8cedec0d37a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T122714244914Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955089408, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 128946465280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955089408, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66955089408, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 5149190656, + "inactive_split_bytes_peak": 14221501952, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 143776546816, + "reserved_bytes_peak": 143776546816, + "segment_count_current": 840, + "segment_count_peak": 840, + "timestamp_utc": "2026-08-01T12:27:14.244464Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955089408, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110792706560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 128946465280, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110792706560, + "device": 7, + "inactive_split_bytes_current": 10757124608, + "inactive_split_bytes_peak": 14221501952, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142883160064, + "reserved_bytes_peak": 142883160064, + "segment_count_current": 834, + "segment_count_peak": 834, + "timestamp_utc": "2026-08-01T12:26:13.623065Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5149190656, + "inactive_split_bytes_current_delta": -5607933952, + "inactive_split_bytes_peak": 14221501952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 143776546816, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 143776546816, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 840, + "segment_count_current_delta": 6, + "segment_count_peak": 840, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T122714244914Z.pickle", + "timestamp_utc": "2026-08-01T12:27:14.332711Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T122714244914Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T122714244914Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0172ef7d3963967f95a90baa1741f5531841d354 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T122714244914Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8037f96ceef30988dc11718d17ae8f94882a8cf71f6561e34bc760c683d93e4 +size 4190960 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..dca7fcefdf7e59a9d3ada2b2d217051aa41162f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 4096 +capacity: 1.00 +expert_dispatch_overlap: off +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6013 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 246 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T123009322595Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T123009322595Z.json new file mode 100644 index 0000000000000000000000000000000000000000..25f2997b1ed46308896b12e3e31f4844fd82a3c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T123009322595Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 87042185216, + "active_bytes_peak_delta": 51575848960, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87042185216, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87042185216, + "device": 0, + "inactive_split_bytes_current": 5265261568, + "inactive_split_bytes_peak": 8333005824, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 95946801152, + "reserved_bytes_peak": 95946801152, + "segment_count_current": 460, + "segment_count_peak": 460, + "timestamp_utc": "2026-08-01T12:30:09.322011Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 87042185216, + "allocated_bytes_peak_delta": 51575848960, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:29:00.741209Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5265261568, + "inactive_split_bytes_current_delta": 5130345472, + "inactive_split_bytes_peak": 8333005824, + "inactive_split_bytes_peak_delta": 8172374016, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 95946801152, + "reserved_bytes_current_delta": 56700698624, + "reserved_bytes_peak": 95946801152, + "reserved_bytes_peak_delta": 56700698624, + "segment_count_current": 460, + "segment_count_current_delta": 388, + "segment_count_peak": 460, + "segment_count_peak_delta": 388, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T123009322595Z.pickle", + "timestamp_utc": "2026-08-01T12:30:09.439652Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T123009322595Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T123009322595Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8d49ec22806541f1b0670f2d3baa26159a04be0b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T123009322595Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0cdfbd23244daf7053f093933cd7e4ab49c19bea25ff3eec6aca10e008b85a3 +size 5949198 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T123010631008Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T123010631008Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf4598b98e6cf3fdac93d9979d23829d08bb7a26 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T123010631008Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110677568000, + "active_bytes_peak_delta": 23635382784, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110677568000, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110677568000, + "device": 0, + "inactive_split_bytes_current": 6685033472, + "inactive_split_bytes_peak": 8333005824, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113040687104, + "reserved_bytes_peak": 113040687104, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.630575Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110677568000, + "allocated_bytes_peak_delta": 23635382784, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87042185216, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87042185216, + "device": 0, + "inactive_split_bytes_current": 5265261568, + "inactive_split_bytes_peak": 8333005824, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 95946801152, + "reserved_bytes_peak": 95946801152, + "segment_count_current": 460, + "segment_count_peak": 460, + "timestamp_utc": "2026-08-01T12:30:09.459885Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6685033472, + "inactive_split_bytes_current_delta": 1419771904, + "inactive_split_bytes_peak": 8333005824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113040687104, + "reserved_bytes_current_delta": 17093885952, + "reserved_bytes_peak": 113040687104, + "reserved_bytes_peak_delta": 17093885952, + "segment_count_current": 541, + "segment_count_current_delta": 81, + "segment_count_peak": 541, + "segment_count_peak_delta": 81, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T123010631008Z.pickle", + "timestamp_utc": "2026-08-01T12:30:10.741097Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T123010631008Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T123010631008Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fbd49b57558697953b9729df5962db4a96a564fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T123010631008Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f25f9252f790ad5a71a4b35b9939d521ddd6b4794119009abe86e9e5b4b005d1 +size 6077618 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T123011865187Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T123011865187Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fdca4390bf9c9440eb4b9aa864be1a2b8ec0446f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T123011865187Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110678220800, + "active_bytes_peak_delta": 652800, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110678220800, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110678220800, + "device": 0, + "inactive_split_bytes_current": 7154795520, + "inactive_split_bytes_peak": 8644687872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113042784256, + "reserved_bytes_peak": 113042784256, + "segment_count_current": 542, + "segment_count_peak": 542, + "timestamp_utc": "2026-08-01T12:30:11.864786Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110678220800, + "allocated_bytes_peak_delta": 652800, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110677568000, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110677568000, + "device": 0, + "inactive_split_bytes_current": 6685033472, + "inactive_split_bytes_peak": 8333005824, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113040687104, + "reserved_bytes_peak": 113040687104, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.757110Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7154795520, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8644687872, + "inactive_split_bytes_peak_delta": 311682048, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113042784256, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 113042784256, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 542, + "segment_count_current_delta": 1, + "segment_count_peak": 542, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T123011865187Z.pickle", + "timestamp_utc": "2026-08-01T12:30:11.975549Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T123011865187Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T123011865187Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e104e0b6848f8b2f8226d08facafe42a02778fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T123011865187Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d27e05da798c621a7b579de3eb255e4826dad3bc2c6cfcdab0d7de496e8c8401 +size 6595845 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T123014829639Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T123014829639Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f9302afa8d7f6d29950dfd194321a0e8099c5066 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T123014829639Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525581312, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 128841779200, + "active_bytes_peak_delta": 18163558400, + "after": { + "active_bytes_current": 72525581312, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 72525581312, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_peak": 15301622272, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_peak": 143195635712, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:30:14.829083Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525581312, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 110690903552, + "allocated_bytes_peak_delta": 12682752, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110678220800, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110678220800, + "device": 0, + "inactive_split_bytes_current": 7154795520, + "inactive_split_bytes_peak": 8644687872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113042784256, + "reserved_bytes_peak": 113042784256, + "segment_count_current": 542, + "segment_count_peak": 542, + "timestamp_utc": "2026-08-01T12:30:11.991007Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_current_delta": 4040028160, + "inactive_split_bytes_peak": 15301622272, + "inactive_split_bytes_peak_delta": 6656934400, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_current_delta": 30152851456, + "reserved_bytes_peak": 143195635712, + "reserved_bytes_peak_delta": 30152851456, + "segment_count_current": 698, + "segment_count_current_delta": 156, + "segment_count_peak": 698, + "segment_count_peak_delta": 156, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T123014829639Z.pickle", + "timestamp_utc": "2026-08-01T12:30:14.954580Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T123014829639Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T123014829639Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2858ca3a9f5d233660155fe0b51d4b64a158e424 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T123014829639Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab171bca0ec141b8d6c3355282aeba55f0f04c10d559c069ffbe36f785c231e9 +size 6321280 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T123016026618Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T123016026618Z.json new file mode 100644 index 0000000000000000000000000000000000000000..714d5ca6d04d5e0101cd54302abcc66a6fbdba57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T123016026618Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525581312, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128841779200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525581312, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 72525581312, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_peak": 15301622272, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_peak": 143195635712, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:30:16.026132Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525581312, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110690903552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525581312, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 72525581312, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_peak": 15301622272, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_peak": 143195635712, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:30:14.972084Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15301622272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 143195635712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 698, + "segment_count_current_delta": 0, + "segment_count_peak": 698, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T123016026618Z.pickle", + "timestamp_utc": "2026-08-01T12:30:16.032274Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T123016026618Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T123016026618Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..673b4b02ae864a37f40f702006dc035a2a9bd25a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T123016026618Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1bec90e81b65938ece5624d88498f7fbd413654124d9ae37df36f61c40a0d45 +size 180106 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T123017082236Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T123017082236Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e00ecd050b96f1d0b5ee67336f995cb446220e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T123017082236Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525581312, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128841779200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525581312, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 72525581312, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_peak": 15301622272, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_peak": 143195635712, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:30:17.081792Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525581312, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110690903552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525581312, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 72525581312, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_peak": 15301622272, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_peak": 143195635712, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:30:16.033185Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15301622272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 143195635712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 698, + "segment_count_current_delta": 0, + "segment_count_peak": 698, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T123017082236Z.pickle", + "timestamp_utc": "2026-08-01T12:30:17.087936Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T123017082236Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T123017082236Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8b78adc7f5dbc75468f9daf8d47aab70803f4b0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T123017082236Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62a214d1f096c096dcfb4b0b4882297fc4d42fe0414dc3a8f677c381f8265a3a +size 180106 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T123018137403Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T123018137403Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67e06e836b8aa449de5fc3b4d10271e2d1894b75 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T123018137403Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525581312, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128841779200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525581312, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 72525581312, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_peak": 15301622272, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_peak": 143195635712, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:30:18.137000Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525581312, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110690903552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525581312, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 72525581312, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_peak": 15301622272, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_peak": 143195635712, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:30:17.088860Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15301622272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 143195635712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 698, + "segment_count_current_delta": 0, + "segment_count_peak": 698, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T123018137403Z.pickle", + "timestamp_utc": "2026-08-01T12:30:18.142374Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T123018137403Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T123018137403Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..666c08a74c0e954b427ac1ecf56c90a736d78cbc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T123018137403Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec538d90d1a63130230a32c41121913227d18177b76327dc24914500bb6495a2 +size 180106 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T123019189083Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T123019189083Z.json new file mode 100644 index 0000000000000000000000000000000000000000..37a065c8f36ab0ab7a04063a52afde8dfd6738d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T123019189083Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525581312, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128841779200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525581312, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 72525581312, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_peak": 15301622272, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_peak": 143195635712, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:30:19.188667Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525581312, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110690903552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525581312, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 72525581312, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_peak": 15301622272, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_peak": 143195635712, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:30:18.143247Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15301622272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 143195635712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 698, + "segment_count_current_delta": 0, + "segment_count_peak": 698, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T123019189083Z.pickle", + "timestamp_utc": "2026-08-01T12:30:19.194349Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T123019189083Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T123019189083Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b603a86f41d0a93629bfe4d1fe4f6eee3f405452 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T123019189083Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efa986f5b53932c31b8854d2824f547e6a7818bbc80ce27e648554a290f40c1c +size 180106 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T123020241034Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T123020241034Z.json new file mode 100644 index 0000000000000000000000000000000000000000..837dc0dbcf35e56b6b432b46a58cf115ccd4f8d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T123020241034Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525581312, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128841779200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525581312, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 72525581312, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_peak": 15301622272, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_peak": 143195635712, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:30:20.240611Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525581312, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110690903552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525581312, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 72525581312, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_peak": 15301622272, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_peak": 143195635712, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:30:19.195266Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15301622272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 143195635712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 698, + "segment_count_current_delta": 0, + "segment_count_peak": 698, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T123020241034Z.pickle", + "timestamp_utc": "2026-08-01T12:30:20.246648Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T123020241034Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T123020241034Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a2d3b2a114240b2c8246b3b55686d3faacf7486 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T123020241034Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbee812e6afdaf6badad5f26566c6b9c61adc920f09b6b8c1a9183ce80304f6c +size 180106 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T123122076634Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T123122076634Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67351cc6279b0ff6ae1520f94c50466463285d8b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T123122076634Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61318861824, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 128841779200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61318861824, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 61318861824, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 7138470912, + "inactive_split_bytes_peak": 15301622272, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 143264841728, + "reserved_bytes_peak": 143264841728, + "segment_count_current": 703, + "segment_count_peak": 703, + "timestamp_utc": "2026-08-01T12:31:22.076159Z", + "world_size": 8 + }, + "allocated_bytes_current": 61318861824, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 110690903552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525581312, + "active_bytes_peak": 128841779200, + "allocated_bytes_current": 72525581312, + "allocated_bytes_peak": 110690903552, + "device": 0, + "inactive_split_bytes_current": 11194823680, + "inactive_split_bytes_peak": 15301622272, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 143195635712, + "reserved_bytes_peak": 143195635712, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:30:20.247559Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7138470912, + "inactive_split_bytes_current_delta": -4056352768, + "inactive_split_bytes_peak": 15301622272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 143264841728, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 143264841728, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 703, + "segment_count_current_delta": 5, + "segment_count_peak": 703, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T123122076634Z.pickle", + "timestamp_utc": "2026-08-01T12:31:22.163492Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T123122076634Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T123122076634Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..65b18e289c293791ddd464f6fa3f08efb074a552 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T123122076634Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:600c9cd6f1b86cedef3557400a96617fb2d0bfb4095b4f2cab8f03889c247fca +size 3968518 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T123009322649Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T123009322649Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e43415395fea2f2e46e0d2893ed6da44fa29ec7b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T123009322649Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 87042705408, + "active_bytes_peak_delta": 51576369152, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87042705408, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87042705408, + "device": 1, + "inactive_split_bytes_current": 4845831168, + "inactive_split_bytes_peak": 8075752448, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 95711920128, + "reserved_bytes_peak": 95711920128, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T12:30:09.322027Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 87042705408, + "allocated_bytes_peak_delta": 51576369152, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:29:00.740408Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4845831168, + "inactive_split_bytes_current_delta": 4713012224, + "inactive_split_bytes_peak": 8075752448, + "inactive_split_bytes_peak_delta": 7915366400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 95711920128, + "reserved_bytes_current_delta": 56467914752, + "reserved_bytes_peak": 95711920128, + "reserved_bytes_peak_delta": 56467914752, + "segment_count_current": 461, + "segment_count_current_delta": 390, + "segment_count_peak": 461, + "segment_count_peak_delta": 390, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T123009322649Z.pickle", + "timestamp_utc": "2026-08-01T12:30:09.442755Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T123009322649Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T123009322649Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da7a4a0db888faec446379d4429da048dd5e6387 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T123009322649Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4215812a5e70c48dfec94d735361d5d8dd70102d5279dde9bf117d0cc3acb32f +size 5928060 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T123010630951Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T123010630951Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9e5cf3a3dd8da771e3b81e795f2169533058819b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T123010630951Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110683463680, + "active_bytes_peak_delta": 23640758272, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 1, + "inactive_split_bytes_current": 6439666688, + "inactive_split_bytes_peak": 8109701120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.630537Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683463680, + "allocated_bytes_peak_delta": 23640758272, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87042705408, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87042705408, + "device": 1, + "inactive_split_bytes_current": 4845831168, + "inactive_split_bytes_peak": 8075752448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 95711920128, + "reserved_bytes_peak": 95711920128, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T12:30:09.463333Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6439666688, + "inactive_split_bytes_current_delta": 1593835520, + "inactive_split_bytes_peak": 8109701120, + "inactive_split_bytes_peak_delta": 33948672, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113034395648, + "reserved_bytes_current_delta": 17322475520, + "reserved_bytes_peak": 113034395648, + "reserved_bytes_peak_delta": 17322475520, + "segment_count_current": 541, + "segment_count_current_delta": 80, + "segment_count_peak": 541, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T123010630951Z.pickle", + "timestamp_utc": "2026-08-01T12:30:10.743729Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T123010630951Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T123010630951Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..87b42df9ff9d3cd006c88bf69a6f65a5e197b874 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T123010630951Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edf9b11a7e2dc478afea1b7fe9ee7f634aa0335ca748c90f47ec457181ac446d +size 6015632 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T123011865179Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T123011865179Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ffa976c25e5ec3b24443e642de571d5269b8e955 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T123011865179Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110683463680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 1, + "inactive_split_bytes_current": 6202688512, + "inactive_split_bytes_peak": 8109701120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:11.864787Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683463680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 1, + "inactive_split_bytes_current": 6439666688, + "inactive_split_bytes_peak": 8109701120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.759679Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6202688512, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 8109701120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113034395648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113034395648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 541, + "segment_count_current_delta": 0, + "segment_count_peak": 541, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T123011865179Z.pickle", + "timestamp_utc": "2026-08-01T12:30:11.981026Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T123011865179Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T123011865179Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..af23aef7aed0f7dad8fd0d9a02cb245cb93ace2c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T123011865179Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:505d98465e54f8b6ca233f70bfdddd588327169101711baea9dfc97167df8b05 +size 6593987 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T123014829680Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T123014829680Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d8b53b85bcac7959d2050aa43106661c9237646 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T123014829680Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 18169325568, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142235140096, + "reserved_bytes_peak": 142235140096, + "segment_count_current": 695, + "segment_count_peak": 695, + "timestamp_utc": "2026-08-01T12:30:14.829112Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 17664000, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 1, + "inactive_split_bytes_current": 6202688512, + "inactive_split_bytes_peak": 8109701120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:11.998924Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 3986419712, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 6594812928, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142235140096, + "reserved_bytes_current_delta": 29200744448, + "reserved_bytes_peak": 142235140096, + "reserved_bytes_peak_delta": 29200744448, + "segment_count_current": 695, + "segment_count_current_delta": 154, + "segment_count_peak": 695, + "segment_count_peak_delta": 154, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T123014829680Z.pickle", + "timestamp_utc": "2026-08-01T12:30:14.954916Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T123014829680Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T123014829680Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f7a43bfbc5be749d1a5f547c0921f9c586a2e95 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T123014829680Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2864bed8956bc43707732c7f7ad49317418ff13fba22aead4f50b73d6a26b705 +size 6191313 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T123016026633Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T123016026633Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2f14d1454f8aa2a385ca743dc9ee8638eb47c281 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T123016026633Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142235140096, + "reserved_bytes_peak": 142235140096, + "segment_count_current": 695, + "segment_count_peak": 695, + "timestamp_utc": "2026-08-01T12:30:16.026141Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142235140096, + "reserved_bytes_peak": 142235140096, + "segment_count_current": 695, + "segment_count_peak": 695, + "timestamp_utc": "2026-08-01T12:30:14.972758Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142235140096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142235140096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 695, + "segment_count_current_delta": 0, + "segment_count_peak": 695, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T123016026633Z.pickle", + "timestamp_utc": "2026-08-01T12:30:16.032222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T123016026633Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T123016026633Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..24034fa96e07c8a4d4fe63fac94c7a13682d4f94 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T123016026633Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7b576dfa1a87317d02bbd95dbd1d11ce43ea912cea8070613bde4001d16f0bc +size 178043 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T123017082230Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T123017082230Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15a9f5de3b7d10a013e289ce25dd380cb235b698 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T123017082230Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142235140096, + "reserved_bytes_peak": 142235140096, + "segment_count_current": 695, + "segment_count_peak": 695, + "timestamp_utc": "2026-08-01T12:30:17.081788Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142235140096, + "reserved_bytes_peak": 142235140096, + "segment_count_current": 695, + "segment_count_peak": 695, + "timestamp_utc": "2026-08-01T12:30:16.033127Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142235140096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142235140096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 695, + "segment_count_current_delta": 0, + "segment_count_peak": 695, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T123017082230Z.pickle", + "timestamp_utc": "2026-08-01T12:30:17.087511Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T123017082230Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T123017082230Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2679902f183a9404eeb3c7332aefdc4c6804ebff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T123017082230Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08ce476bd7bffdc413118569ee1c2b6e8b6f42148046e3f50454e039aa180cf4 +size 178074 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T123018137394Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T123018137394Z.json new file mode 100644 index 0000000000000000000000000000000000000000..53adddc39dff32182d513ff7905f76c0e528c2f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T123018137394Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142235140096, + "reserved_bytes_peak": 142235140096, + "segment_count_current": 695, + "segment_count_peak": 695, + "timestamp_utc": "2026-08-01T12:30:18.136984Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142235140096, + "reserved_bytes_peak": 142235140096, + "segment_count_current": 695, + "segment_count_peak": 695, + "timestamp_utc": "2026-08-01T12:30:17.088406Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142235140096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142235140096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 695, + "segment_count_current_delta": 0, + "segment_count_peak": 695, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T123018137394Z.pickle", + "timestamp_utc": "2026-08-01T12:30:18.142218Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T123018137394Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T123018137394Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..935f4e20e733258d46ee022dbe38b7b85211c115 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T123018137394Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79917a999bd524b2bfa07dcfe912465f3bed9200ca69312d2b0ba5213d2d4692 +size 178043 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T123019189111Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T123019189111Z.json new file mode 100644 index 0000000000000000000000000000000000000000..831b5f3b4aa6a940a6e1a6476e02dd419154e2dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T123019189111Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142235140096, + "reserved_bytes_peak": 142235140096, + "segment_count_current": 695, + "segment_count_peak": 695, + "timestamp_utc": "2026-08-01T12:30:19.188676Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142235140096, + "reserved_bytes_peak": 142235140096, + "segment_count_current": 695, + "segment_count_peak": 695, + "timestamp_utc": "2026-08-01T12:30:18.143069Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142235140096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142235140096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 695, + "segment_count_current_delta": 0, + "segment_count_peak": 695, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T123019189111Z.pickle", + "timestamp_utc": "2026-08-01T12:30:19.194389Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T123019189111Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T123019189111Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c8d3e92b8f8634fd6041efe9aaf53784f267a40 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T123019189111Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9aa461fe6cdad46abf4523bd84adff33837c389c87ac8a02efccf5b0b85b9035 +size 178074 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T123020241078Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T123020241078Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ee24580e3003f5eaaa75e6d8bef4ffe1552dd44 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T123020241078Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142235140096, + "reserved_bytes_peak": 142235140096, + "segment_count_current": 695, + "segment_count_peak": 695, + "timestamp_utc": "2026-08-01T12:30:20.240629Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142235140096, + "reserved_bytes_peak": 142235140096, + "segment_count_current": 695, + "segment_count_peak": 695, + "timestamp_utc": "2026-08-01T12:30:19.195310Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142235140096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142235140096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 695, + "segment_count_current_delta": 0, + "segment_count_peak": 695, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T123020241078Z.pickle", + "timestamp_utc": "2026-08-01T12:30:20.246497Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T123020241078Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T123020241078Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c9578aa897a2efea457d72399d988ebbb1c0a757 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T123020241078Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f4f4e0f61906f95b51b8376864d239afc493e9ea97d47f442f7f2814397722f +size 178043 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T123122076634Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T123122076634Z.json new file mode 100644 index 0000000000000000000000000000000000000000..91159b58799459f908e109b60a53bba2c9bff462 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T123122076634Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61317944320, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61317944320, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 61317944320, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 6076132352, + "inactive_split_bytes_peak": 14704514048, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 142304346112, + "reserved_bytes_peak": 142304346112, + "segment_count_current": 700, + "segment_count_peak": 700, + "timestamp_utc": "2026-08-01T12:31:22.076132Z", + "world_size": 8 + }, + "allocated_bytes_current": 61317944320, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 1, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 142235140096, + "reserved_bytes_peak": 142235140096, + "segment_count_current": 695, + "segment_count_peak": 695, + "timestamp_utc": "2026-08-01T12:30:20.247382Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6076132352, + "inactive_split_bytes_current_delta": -4112975872, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 142304346112, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 142304346112, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 700, + "segment_count_current_delta": 5, + "segment_count_peak": 700, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T123122076634Z.pickle", + "timestamp_utc": "2026-08-01T12:31:22.157331Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T123122076634Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T123122076634Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e509f268aca3b7c9c4ef5a6e0b21d9a9f094cc9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T123122076634Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39cfa5e6567276fd35184179f447302cf207dc2b447a1e3edf274cae1ce47be4 +size 3841420 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T123009322608Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T123009322608Z.json new file mode 100644 index 0000000000000000000000000000000000000000..827ec6d353aafd8154442ee3029bc3905d4355d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T123009322608Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 87042705408, + "active_bytes_peak_delta": 51576369152, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87042705408, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87042705408, + "device": 2, + "inactive_split_bytes_current": 4843734016, + "inactive_split_bytes_peak": 8077849600, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 95711920128, + "reserved_bytes_peak": 95711920128, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T12:30:09.322021Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 87042705408, + "allocated_bytes_peak_delta": 51576369152, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:29:00.740398Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4843734016, + "inactive_split_bytes_current_delta": 4710915072, + "inactive_split_bytes_peak": 8077849600, + "inactive_split_bytes_peak_delta": 7917463552, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 95711920128, + "reserved_bytes_current_delta": 56467914752, + "reserved_bytes_peak": 95711920128, + "reserved_bytes_peak_delta": 56467914752, + "segment_count_current": 461, + "segment_count_current_delta": 390, + "segment_count_peak": 461, + "segment_count_peak_delta": 390, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T123009322608Z.pickle", + "timestamp_utc": "2026-08-01T12:30:09.445831Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T123009322608Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T123009322608Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7293a594268b24d860a0f7702520696c59e7638c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T123009322608Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc37d2a67f399ea67a108ace4f225765b196f37b9c45397d9f4eea1e632713f6 +size 5940613 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T123010630968Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T123010630968Z.json new file mode 100644 index 0000000000000000000000000000000000000000..50c3d2fad2b50621052c783b12f9ded7dd90fc11 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T123010630968Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110683463680, + "active_bytes_peak_delta": 23640758272, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 2, + "inactive_split_bytes_current": 6435472384, + "inactive_split_bytes_peak": 8105506816, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.630558Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683463680, + "allocated_bytes_peak_delta": 23640758272, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87042705408, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87042705408, + "device": 2, + "inactive_split_bytes_current": 4843734016, + "inactive_split_bytes_peak": 8077849600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 95711920128, + "reserved_bytes_peak": 95711920128, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T12:30:09.464555Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6435472384, + "inactive_split_bytes_current_delta": 1591738368, + "inactive_split_bytes_peak": 8105506816, + "inactive_split_bytes_peak_delta": 27657216, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 113034395648, + "reserved_bytes_current_delta": 17322475520, + "reserved_bytes_peak": 113034395648, + "reserved_bytes_peak_delta": 17322475520, + "segment_count_current": 541, + "segment_count_current_delta": 80, + "segment_count_peak": 541, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T123010630968Z.pickle", + "timestamp_utc": "2026-08-01T12:30:10.744941Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T123010630968Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T123010630968Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..128fd97c17f7af4fc86c781a627b3827a047b5a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T123010630968Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dfa1aaf52aded9c8823b592f7257752a8bc30c97142113934948a5739e98d17 +size 6015366 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T123011865166Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T123011865166Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8def66b70f11987615ccdbfa5554f7f5d6cb43f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T123011865166Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110683463680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 2, + "inactive_split_bytes_current": 6198494208, + "inactive_split_bytes_peak": 8105506816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:11.864771Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683463680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 2, + "inactive_split_bytes_current": 6435472384, + "inactive_split_bytes_peak": 8105506816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.760877Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6198494208, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 8105506816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 113034395648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113034395648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 541, + "segment_count_current_delta": 0, + "segment_count_peak": 541, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T123011865166Z.pickle", + "timestamp_utc": "2026-08-01T12:30:11.990756Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T123011865166Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T123011865166Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dd14dd993966222ce07fc1ae486768ca0746a132 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T123011865166Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8f9cd6c50cde5034d3a8fd1ab9b726278f49485de8fb77e5faf2fb603de16ae +size 6581926 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T123014829637Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T123014829637Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed4e7e98123ad969dbd071b93d2dcb523d39cb06 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T123014829637Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 18169325568, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:14.829091Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 17664000, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 2, + "inactive_split_bytes_current": 6198494208, + "inactive_split_bytes_peak": 8105506816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:12.009878Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 3990614016, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 6599007232, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 29196550144, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 29196550144, + "segment_count_current": 693, + "segment_count_current_delta": 152, + "segment_count_peak": 693, + "segment_count_peak_delta": 152, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T123014829637Z.pickle", + "timestamp_utc": "2026-08-01T12:30:14.953968Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T123014829637Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T123014829637Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6af1a96ee8c0ff6748155cea30a609074de9c777 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T123014829637Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e150c4a20224da5e2d50e87703f972161da9d103917f9c91ca7d48453526fc2 +size 6251605 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T123016026595Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T123016026595Z.json new file mode 100644 index 0000000000000000000000000000000000000000..84e077ae60cf9477927d49bbb7bb5688c6e4ad8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T123016026595Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:16.026122Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:14.971745Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T123016026595Z.pickle", + "timestamp_utc": "2026-08-01T12:30:16.032056Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T123016026595Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T123016026595Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9429fe27dbc0a24eb79073de23c02e0c328c2c7f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T123016026595Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d255a2e7e87e437fa0f64f592fc3807ef3c817d58853dcf1bcc50c9208e05f70 +size 178721 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T123017082231Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T123017082231Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d910d914a9a6665211a06b61120c147566fa644 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T123017082231Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:17.081796Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:16.032981Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T123017082231Z.pickle", + "timestamp_utc": "2026-08-01T12:30:17.087566Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T123017082231Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T123017082231Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf7256b06df5b1f925a0366253fb301e751a1f99 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T123017082231Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17ee44d7f29bab1b5fadd1f2b7a4e5d30c3404ed08eddbce4bf31fb72b292b7c +size 178721 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T123018137381Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T123018137381Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db101f4f8e5686bec4ecb754b06a7dd80ea6c6e1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T123018137381Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:18.136982Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:17.088484Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T123018137381Z.pickle", + "timestamp_utc": "2026-08-01T12:30:18.142129Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T123018137381Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T123018137381Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..17bf7086f6d41d05e242a55d7be08ce74ce2b7c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T123018137381Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a1f9dd3c81053cb7de9e6731f0bdd0263e75b2152ab9bcbd83985aa6da0505f +size 178721 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T123019189106Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T123019189106Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0fe8bc8df98c3907276f4009f85f5145e0423cf3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T123019189106Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:19.188676Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:18.143083Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T123019189106Z.pickle", + "timestamp_utc": "2026-08-01T12:30:19.194100Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T123019189106Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T123019189106Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..780889d3ce913c67aaa9d1067df2b986be4ca29d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T123019189106Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b531f9648faaa4c7e1edad64f32fae0ba0a60aea7926c2ebaffd3b14daf12ab7 +size 178721 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T123020241045Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T123020241045Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b0620f26808d4f125678330111f0e1778348e0d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T123020241045Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:20.240614Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:19.195040Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T123020241045Z.pickle", + "timestamp_utc": "2026-08-01T12:30:20.246409Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T123020241045Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T123020241045Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e13af4de7cbb426036fec03fb7eb0400fd90be64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T123020241045Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30ffbcd09f7c8e49c76fc12be23f7696e3083ea1465c122dfd48382e9b9888cf +size 178721 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T123122076637Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T123122076637Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3308c8968f70163001cd4d6c3fb19b0f773551a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T123122076637Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61317944320, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61317944320, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 61317944320, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 6076132352, + "inactive_split_bytes_peak": 14704514048, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 142300151808, + "reserved_bytes_peak": 142300151808, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:31:22.076149Z", + "world_size": 8 + }, + "allocated_bytes_current": 61317944320, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 2, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:20.247328Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6076132352, + "inactive_split_bytes_current_delta": -4112975872, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 142300151808, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 142300151808, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 698, + "segment_count_current_delta": 5, + "segment_count_peak": 698, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T123122076637Z.pickle", + "timestamp_utc": "2026-08-01T12:31:22.158639Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T123122076637Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T123122076637Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9df2c5e0f85e2d162bc9a088e09c9fe767d4cb8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T123122076637Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71fe96b343bc3b15187771604ad7e631b0adb28426452a2f9173287a7b8579c9 +size 3989978 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T123009322835Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T123009322835Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a6ec4dee36ef0b69988654a5996fde0737e77434 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T123009322835Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 87042705408, + "active_bytes_peak_delta": 51576369152, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87042705408, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87042705408, + "device": 3, + "inactive_split_bytes_current": 4843734016, + "inactive_split_bytes_peak": 8077849600, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 95711920128, + "reserved_bytes_peak": 95711920128, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T12:30:09.321990Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 87042705408, + "allocated_bytes_peak_delta": 51576369152, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:29:00.740327Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4843734016, + "inactive_split_bytes_current_delta": 4710915072, + "inactive_split_bytes_peak": 8077849600, + "inactive_split_bytes_peak_delta": 7917463552, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 95711920128, + "reserved_bytes_current_delta": 56467914752, + "reserved_bytes_peak": 95711920128, + "reserved_bytes_peak_delta": 56467914752, + "segment_count_current": 461, + "segment_count_current_delta": 390, + "segment_count_peak": 461, + "segment_count_peak_delta": 390, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T123009322835Z.pickle", + "timestamp_utc": "2026-08-01T12:30:09.438764Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T123009322835Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T123009322835Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..989f96d64302c77bc191f6d815571c02d3f0b05a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T123009322835Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2907c8c0c3eab1167b7ac3992b2d78afe6b75a228ea9415ca36ce19b59a673d6 +size 5940613 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T123010630944Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T123010630944Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a0d07a446e7f4ae9378f11414e8bd80317e4890d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T123010630944Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110683463680, + "active_bytes_peak_delta": 23640758272, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 3, + "inactive_split_bytes_current": 6435472384, + "inactive_split_bytes_peak": 8107603968, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.630522Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683463680, + "allocated_bytes_peak_delta": 23640758272, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87042705408, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87042705408, + "device": 3, + "inactive_split_bytes_current": 4843734016, + "inactive_split_bytes_peak": 8077849600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 95711920128, + "reserved_bytes_peak": 95711920128, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T12:30:09.455792Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6435472384, + "inactive_split_bytes_current_delta": 1591738368, + "inactive_split_bytes_peak": 8107603968, + "inactive_split_bytes_peak_delta": 29754368, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 113034395648, + "reserved_bytes_current_delta": 17322475520, + "reserved_bytes_peak": 113034395648, + "reserved_bytes_peak_delta": 17322475520, + "segment_count_current": 541, + "segment_count_current_delta": 80, + "segment_count_peak": 541, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T123010630944Z.pickle", + "timestamp_utc": "2026-08-01T12:30:10.743279Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T123010630944Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T123010630944Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e53885466a152a9b750db9dc81845e2c4dc2f84 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T123010630944Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:794b1d410bc8844b3a560428a4044d6e3bf822bba22a63b732af03b667d8b181 +size 5924171 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T123011865300Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T123011865300Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c97d646213f3806e5571d531f6843831254d6139 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T123011865300Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110683463680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 3, + "inactive_split_bytes_current": 6198494208, + "inactive_split_bytes_peak": 8107603968, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:11.864884Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683463680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 3, + "inactive_split_bytes_current": 6435472384, + "inactive_split_bytes_peak": 8107603968, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.757076Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6198494208, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 8107603968, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 113034395648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113034395648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 541, + "segment_count_current_delta": 0, + "segment_count_peak": 541, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T123011865300Z.pickle", + "timestamp_utc": "2026-08-01T12:30:11.980004Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T123011865300Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T123011865300Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f7a9a037cb8ca8b9f2add6c0ad6e56a77cd9847d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T123011865300Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db7a1425be986a09dbdd1ac8d712db0a4f56cc8421484f8fcde4da50d2f2b482 +size 6593976 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T123014829592Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T123014829592Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bd6c9db435e16116929e05f8b8dc10156920dbb4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T123014829592Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 18169325568, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:14.829066Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 17664000, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 3, + "inactive_split_bytes_current": 6198494208, + "inactive_split_bytes_peak": 8107603968, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:11.995858Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 3990614016, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 6596910080, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 29196550144, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 29196550144, + "segment_count_current": 693, + "segment_count_current_delta": 152, + "segment_count_peak": 693, + "segment_count_peak_delta": 152, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T123014829592Z.pickle", + "timestamp_utc": "2026-08-01T12:30:14.951888Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T123014829592Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T123014829592Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..28119594eab7021057459356ba675eaeb2bdf6f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T123014829592Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:744e9a7cb8c8582e6ee5dbdf4c644bf7dce209ad2d378f40f022163fa6ce0bb9 +size 6294995 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T123016026556Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T123016026556Z.json new file mode 100644 index 0000000000000000000000000000000000000000..74759627860f3d061c00ec1f39271787a2a1e3ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T123016026556Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:16.026095Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:14.967412Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T123016026556Z.pickle", + "timestamp_utc": "2026-08-01T12:30:16.032009Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T123016026556Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T123016026556Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e80865822682d0a20b5447e56d414e9693373c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T123016026556Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fdd56076ffe85d9244fc7a82f0325108cf9e91c670913aeb17e1724a58c9d85 +size 178728 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T123017082383Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T123017082383Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cca1acdf268d282f6aaacac9847bee79193869fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T123017082383Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:17.081857Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:16.032926Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T123017082383Z.pickle", + "timestamp_utc": "2026-08-01T12:30:17.088701Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T123017082383Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T123017082383Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80543deabe917ec3a7daeee696ea479831c294d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T123017082383Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5fa0353922189d5ccf71ddee0e602c0fe59cd955c900026a8a69b1c9c6be301 +size 179475 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T123018137410Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T123018137410Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f33364775fc2175ca00436fb1fb4bd1e0498d949 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T123018137410Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:18.136975Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:17.089663Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T123018137410Z.pickle", + "timestamp_utc": "2026-08-01T12:30:18.142634Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T123018137410Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T123018137410Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2bdd19a69780c65eb1dfc0bc9876e91833020574 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T123018137410Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f9c4fbd8aa7a759dd5059472880f998906668a5e72bf8b40510c4012d0a08db +size 178728 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T123019189073Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T123019189073Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e17271b0a25ef5e7ba86c077391fb0714995ab3a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T123019189073Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:19.188648Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:18.143516Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T123019189073Z.pickle", + "timestamp_utc": "2026-08-01T12:30:19.194235Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T123019189073Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T123019189073Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba02d6c60426436f315c46866202adf032767d6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T123019189073Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e65e6d8990ae374327496b944f9894bb428ec87bef6a62b365990ab2315b8de0 +size 179475 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T123020241016Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T123020241016Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f00912049e2eb33453f84fe665dd25836718e09 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T123020241016Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:20.240574Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:19.195159Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T123020241016Z.pickle", + "timestamp_utc": "2026-08-01T12:30:20.246562Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T123020241016Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T123020241016Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e6f45ddf8c94c8fcd9fcbc611b585af61a8a5d4c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T123020241016Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff023a5e0307eb898db59cdb83581ff41417e9078c520a902384143231c50d32 +size 178728 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T123122076619Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T123122076619Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e0f5895f91934680bdd4fdcf0d4e760ea4a5c80 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T123122076619Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61317944320, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61317944320, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 61317944320, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 6076132352, + "inactive_split_bytes_peak": 14704514048, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 142300151808, + "reserved_bytes_peak": 142300151808, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:31:22.076134Z", + "world_size": 8 + }, + "allocated_bytes_current": 61317944320, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 3, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:20.247453Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6076132352, + "inactive_split_bytes_current_delta": -4112975872, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 142300151808, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 142300151808, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 698, + "segment_count_current_delta": 5, + "segment_count_peak": 698, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T123122076619Z.pickle", + "timestamp_utc": "2026-08-01T12:31:22.157625Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T123122076619Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T123122076619Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44c01bdc3dabf98b7703c82c9321fc938a6100fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T123122076619Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:523466482406492a735af4b37af40fd1c74fb15e175c937eb15a3c267c91f517 +size 3989978 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T123009322564Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T123009322564Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c9378b006269e5b626264a2f6614d550523459a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T123009322564Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508521472, + "active_bytes_current_delta": 37276623872, + "active_bytes_peak": 87048486400, + "active_bytes_peak_delta": 51581707776, + "after": { + "active_bytes_current": 72508521472, + "active_bytes_peak": 87048486400, + "allocated_bytes_current": 72508521472, + "allocated_bytes_peak": 87048486400, + "device": 4, + "inactive_split_bytes_current": 4607951872, + "inactive_split_bytes_peak": 8801276928, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 95651102720, + "reserved_bytes_peak": 95651102720, + "segment_count_current": 459, + "segment_count_peak": 459, + "timestamp_utc": "2026-08-01T12:30:09.321979Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508521472, + "allocated_bytes_current_delta": 37276623872, + "allocated_bytes_peak": 87048486400, + "allocated_bytes_peak_delta": 51581707776, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:29:00.741013Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4607951872, + "inactive_split_bytes_current_delta": 4475575296, + "inactive_split_bytes_peak": 8801276928, + "inactive_split_bytes_peak_delta": 8647867392, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 95651102720, + "reserved_bytes_current_delta": 56407097344, + "reserved_bytes_peak": 95651102720, + "reserved_bytes_peak_delta": 56407097344, + "segment_count_current": 459, + "segment_count_current_delta": 388, + "segment_count_peak": 459, + "segment_count_peak_delta": 388, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T123009322564Z.pickle", + "timestamp_utc": "2026-08-01T12:30:09.441279Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T123009322564Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T123009322564Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6480ff39e5b636fff92bf8414aa3c29279591264 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T123009322564Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee1f132754b66079d4d6826eed2a63fb7a3da512c81557ac6e6ebb768ff1dcb +size 5948684 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T123010630959Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T123010630959Z.json new file mode 100644 index 0000000000000000000000000000000000000000..992ff68cc79992f789a4003421a795922b55f7fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T123010630959Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508521472, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110687758336, + "active_bytes_peak_delta": 23639271936, + "after": { + "active_bytes_current": 72508521472, + "active_bytes_peak": 110687758336, + "allocated_bytes_current": 72508521472, + "allocated_bytes_peak": 110687758336, + "device": 4, + "inactive_split_bytes_current": 6258410496, + "inactive_split_bytes_peak": 8801276928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112916955136, + "reserved_bytes_peak": 112916955136, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:30:10.630538Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508521472, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110687758336, + "allocated_bytes_peak_delta": 23639271936, + "before": { + "active_bytes_current": 72508521472, + "active_bytes_peak": 87048486400, + "allocated_bytes_current": 72508521472, + "allocated_bytes_peak": 87048486400, + "device": 4, + "inactive_split_bytes_current": 4607951872, + "inactive_split_bytes_peak": 8801276928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 95651102720, + "reserved_bytes_peak": 95651102720, + "segment_count_current": 459, + "segment_count_peak": 459, + "timestamp_utc": "2026-08-01T12:30:09.460081Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6258410496, + "inactive_split_bytes_current_delta": 1650458624, + "inactive_split_bytes_peak": 8801276928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112916955136, + "reserved_bytes_current_delta": 17265852416, + "reserved_bytes_peak": 112916955136, + "reserved_bytes_peak_delta": 17265852416, + "segment_count_current": 543, + "segment_count_current_delta": 84, + "segment_count_peak": 543, + "segment_count_peak_delta": 84, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T123010630959Z.pickle", + "timestamp_utc": "2026-08-01T12:30:10.748441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T123010630959Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T123010630959Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2b23f1982dc79ae15b1ee9cdd00a02799481db65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T123010630959Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ed8b23a9efcfe8c14837307cdbddd118a5e8d3c013b2416d8b6ffa97dd24c66 +size 6611859 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T123011865142Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T123011865142Z.json new file mode 100644 index 0000000000000000000000000000000000000000..de8f1325394c2a5b65e54e46ca7b8efc0230b2c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T123011865142Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508521472, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110687758336, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72508521472, + "active_bytes_peak": 110687758336, + "allocated_bytes_current": 72508521472, + "allocated_bytes_peak": 110687758336, + "device": 4, + "inactive_split_bytes_current": 5551670272, + "inactive_split_bytes_peak": 8801276928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112916955136, + "reserved_bytes_peak": 112916955136, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:30:11.864748Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508521472, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110687758336, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72508521472, + "active_bytes_peak": 110687758336, + "allocated_bytes_current": 72508521472, + "allocated_bytes_peak": 110687758336, + "device": 4, + "inactive_split_bytes_current": 6258410496, + "inactive_split_bytes_peak": 8801276928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112916955136, + "reserved_bytes_peak": 112916955136, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:30:10.764570Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5551670272, + "inactive_split_bytes_current_delta": -706740224, + "inactive_split_bytes_peak": 8801276928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112916955136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112916955136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 543, + "segment_count_current_delta": 0, + "segment_count_peak": 543, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T123011865142Z.pickle", + "timestamp_utc": "2026-08-01T12:30:11.981659Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T123011865142Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T123011865142Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf91a82a200873ac7fe04d36af26055b2ceffef6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T123011865142Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8afad2630c6919135a89848f68e7e7007b778926652719a71be45ce7920f9d48 +size 6614426 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T123014829596Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T123014829596Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d78236a56c3b268bb717ad53643f32fbf9200b87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T123014829596Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525564928, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128840321024, + "active_bytes_peak_delta": 18152562688, + "after": { + "active_bytes_current": 72525564928, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 72525564928, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_peak": 13367802368, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141643743232, + "reserved_bytes_peak": 141643743232, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-08-01T12:30:14.829032Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525564928, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110689445376, + "allocated_bytes_peak_delta": 1687040, + "before": { + "active_bytes_current": 72508521472, + "active_bytes_peak": 110687758336, + "allocated_bytes_current": 72508521472, + "allocated_bytes_peak": 110687758336, + "device": 4, + "inactive_split_bytes_current": 5551670272, + "inactive_split_bytes_peak": 8801276928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112916955136, + "reserved_bytes_peak": 112916955136, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:30:11.997398Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_current_delta": 3158044672, + "inactive_split_bytes_peak": 13367802368, + "inactive_split_bytes_peak_delta": 4566525440, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141643743232, + "reserved_bytes_current_delta": 28726788096, + "reserved_bytes_peak": 141643743232, + "reserved_bytes_peak_delta": 28726788096, + "segment_count_current": 697, + "segment_count_current_delta": 154, + "segment_count_peak": 697, + "segment_count_peak_delta": 154, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T123014829596Z.pickle", + "timestamp_utc": "2026-08-01T12:30:14.956855Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T123014829596Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T123014829596Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5866df6c3da81b925e936369672588802fa057a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T123014829596Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfd49a242f081d10fe7ca7f9535d956cb2623a72b2a33b38b6c4fed0d4017197 +size 6374110 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T123016026603Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T123016026603Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d403e41fa21386e3e524f452a8a4bb24ccd5f6c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T123016026603Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525564928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128840321024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525564928, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 72525564928, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_peak": 13367802368, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141643743232, + "reserved_bytes_peak": 141643743232, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-08-01T12:30:16.026098Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525564928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110689445376, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525564928, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 72525564928, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_peak": 13367802368, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 141643743232, + "reserved_bytes_peak": 141643743232, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-08-01T12:30:14.974243Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13367802368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141643743232, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141643743232, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 697, + "segment_count_current_delta": 0, + "segment_count_peak": 697, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T123016026603Z.pickle", + "timestamp_utc": "2026-08-01T12:30:16.032324Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T123016026603Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T123016026603Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ed04c72bf72f43132ab10b9886ed044f5f29908 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T123016026603Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e8d56bc98633c6cea1c4df04c7af90e196bcf382a8a9e357be089a2213bdcb5 +size 182050 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T123017082305Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T123017082305Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d9b1f852fae7d91c8fa71bd530c5f200544cca83 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T123017082305Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525564928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128840321024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525564928, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 72525564928, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_peak": 13367802368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141643743232, + "reserved_bytes_peak": 141643743232, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-08-01T12:30:17.081797Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525564928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110689445376, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525564928, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 72525564928, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_peak": 13367802368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 141643743232, + "reserved_bytes_peak": 141643743232, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-08-01T12:30:16.033241Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13367802368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141643743232, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141643743232, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 697, + "segment_count_current_delta": 0, + "segment_count_peak": 697, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T123017082305Z.pickle", + "timestamp_utc": "2026-08-01T12:30:17.088446Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T123017082305Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T123017082305Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a7eff07d711bf257bff7a9598f73503078530a13 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T123017082305Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a49398da57da709d1b3c9274d08fc12eac2dd654ea62ff896c0152fbedd82061 +size 182050 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T123018137334Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T123018137334Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e88785de7b15f1de08d9946fccfc91f1223b7108 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T123018137334Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525564928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128840321024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525564928, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 72525564928, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_peak": 13367802368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141643743232, + "reserved_bytes_peak": 141643743232, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-08-01T12:30:18.136934Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525564928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110689445376, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525564928, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 72525564928, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_peak": 13367802368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 141643743232, + "reserved_bytes_peak": 141643743232, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-08-01T12:30:17.089390Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13367802368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141643743232, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141643743232, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 697, + "segment_count_current_delta": 0, + "segment_count_peak": 697, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T123018137334Z.pickle", + "timestamp_utc": "2026-08-01T12:30:18.142436Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T123018137334Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T123018137334Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e4ccab4f74bd7edcbbdbafeb6d7949734970ad6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T123018137334Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbffb173d4c10a7ef5fb37d51b6f3d0ab9c17baeeae92722b131e0df013a2134 +size 182050 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T123019189106Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T123019189106Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9dfdb026f1a011abb207c0bdb6ace8d28bda512c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T123019189106Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525564928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128840321024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525564928, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 72525564928, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_peak": 13367802368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141643743232, + "reserved_bytes_peak": 141643743232, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-08-01T12:30:19.188674Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525564928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110689445376, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525564928, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 72525564928, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_peak": 13367802368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 141643743232, + "reserved_bytes_peak": 141643743232, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-08-01T12:30:18.143329Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13367802368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141643743232, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141643743232, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 697, + "segment_count_current_delta": 0, + "segment_count_peak": 697, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T123019189106Z.pickle", + "timestamp_utc": "2026-08-01T12:30:19.194866Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T123019189106Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T123019189106Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ff448472afcae70391e618f465795d30eb79eda --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T123019189106Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73333446b4f4fc6ad50ae8fe1e17ca90000f7a97734331ded0207c7b34619799 +size 182050 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T123020240945Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T123020240945Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9950569a987d721235eece09bf3d54c694711189 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T123020240945Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525564928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128840321024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525564928, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 72525564928, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_peak": 13367802368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141643743232, + "reserved_bytes_peak": 141643743232, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-08-01T12:30:20.240546Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525564928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110689445376, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525564928, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 72525564928, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_peak": 13367802368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 141643743232, + "reserved_bytes_peak": 141643743232, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-08-01T12:30:19.195830Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13367802368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141643743232, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 141643743232, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 697, + "segment_count_current_delta": 0, + "segment_count_peak": 697, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T123020240945Z.pickle", + "timestamp_utc": "2026-08-01T12:30:20.246221Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T123020240945Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T123020240945Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ad80941b745216a706de9a25b8eb6857088a0e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T123020240945Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:917cee84884b269c7732579fbdc3608d7629bb450d7e58cc6a9f7e5e515d3344 +size 182050 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T123122076652Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T123122076652Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a46b26513728d20655b6664095c192f13eb5e0eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T123122076652Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61318845440, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 128840321024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61318845440, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 61318845440, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 4888243200, + "inactive_split_bytes_peak": 13367802368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 141712949248, + "reserved_bytes_peak": 141712949248, + "segment_count_current": 702, + "segment_count_peak": 702, + "timestamp_utc": "2026-08-01T12:31:22.076169Z", + "world_size": 8 + }, + "allocated_bytes_current": 61318845440, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 110689445376, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525564928, + "active_bytes_peak": 128840321024, + "allocated_bytes_current": 72525564928, + "allocated_bytes_peak": 110689445376, + "device": 4, + "inactive_split_bytes_current": 8709714944, + "inactive_split_bytes_peak": 13367802368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 141643743232, + "reserved_bytes_peak": 141643743232, + "segment_count_current": 697, + "segment_count_peak": 697, + "timestamp_utc": "2026-08-01T12:30:20.247122Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4888243200, + "inactive_split_bytes_current_delta": -3821471744, + "inactive_split_bytes_peak": 13367802368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 141712949248, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 141712949248, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 702, + "segment_count_current_delta": 5, + "segment_count_peak": 702, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T123122076652Z.pickle", + "timestamp_utc": "2026-08-01T12:31:22.161781Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T123122076652Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T123122076652Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b47f3ea0f23a65f63b4de955dac6d8db38b0db4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T123122076652Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f5a8052b0847e61849ef801df12edaf3035ea760d28b03f0ea7c2aa2d8449b6 +size 3969707 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T123009322592Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T123009322592Z.json new file mode 100644 index 0000000000000000000000000000000000000000..02dc41e12f4f95a0f9b5660c186018769b8bc741 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T123009322592Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 87042705408, + "active_bytes_peak_delta": 51576369152, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87042705408, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87042705408, + "device": 5, + "inactive_split_bytes_current": 4845831168, + "inactive_split_bytes_peak": 8073655296, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 95711920128, + "reserved_bytes_peak": 95711920128, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T12:30:09.321975Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 87042705408, + "allocated_bytes_peak_delta": 51576369152, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:29:00.740573Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4845831168, + "inactive_split_bytes_current_delta": 4713012224, + "inactive_split_bytes_peak": 8073655296, + "inactive_split_bytes_peak_delta": 7913269248, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 95711920128, + "reserved_bytes_current_delta": 56467914752, + "reserved_bytes_peak": 95711920128, + "reserved_bytes_peak_delta": 56467914752, + "segment_count_current": 461, + "segment_count_current_delta": 390, + "segment_count_peak": 461, + "segment_count_peak_delta": 390, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T123009322592Z.pickle", + "timestamp_utc": "2026-08-01T12:30:09.443723Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T123009322592Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T123009322592Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b94ffc285f3ab77a0074e000ea5cc58b6bade8b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T123009322592Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:105a8e4a01992de82418437e243b4de9c117f9a06f48ad12270b6cc938b19aee +size 5940611 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T123010630956Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T123010630956Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5fe1688726a9ffc3934f1a8a566ad8f860d58f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T123010630956Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110683463680, + "active_bytes_peak_delta": 23640758272, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 5, + "inactive_split_bytes_current": 6433375232, + "inactive_split_bytes_peak": 8109701120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.630529Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683463680, + "allocated_bytes_peak_delta": 23640758272, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87042705408, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87042705408, + "device": 5, + "inactive_split_bytes_current": 4845831168, + "inactive_split_bytes_peak": 8073655296, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 95711920128, + "reserved_bytes_peak": 95711920128, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T12:30:09.463258Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6433375232, + "inactive_split_bytes_current_delta": 1587544064, + "inactive_split_bytes_peak": 8109701120, + "inactive_split_bytes_peak_delta": 36045824, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113034395648, + "reserved_bytes_current_delta": 17322475520, + "reserved_bytes_peak": 113034395648, + "reserved_bytes_peak_delta": 17322475520, + "segment_count_current": 541, + "segment_count_current_delta": 80, + "segment_count_peak": 541, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T123010630956Z.pickle", + "timestamp_utc": "2026-08-01T12:30:10.746515Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T123010630956Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T123010630956Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..54e87b59b6caf084b22700e0019214a449205ede --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T123010630956Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:937c9da75f1e9b703cc427985c5ddb357eb9fb20db2a835762b7f34d27404a64 +size 5922786 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T123011865142Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T123011865142Z.json new file mode 100644 index 0000000000000000000000000000000000000000..860e3ac0437e800637f31b69b23be507db0baa30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T123011865142Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110683463680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 5, + "inactive_split_bytes_current": 6200591360, + "inactive_split_bytes_peak": 8109701120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:11.864736Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683463680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 5, + "inactive_split_bytes_current": 6433375232, + "inactive_split_bytes_peak": 8109701120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.763253Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6200591360, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 8109701120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113034395648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113034395648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 541, + "segment_count_current_delta": 0, + "segment_count_peak": 541, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T123011865142Z.pickle", + "timestamp_utc": "2026-08-01T12:30:11.981670Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T123011865142Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T123011865142Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..31c483da1b9b38577a31e8bc7b03ae173ecc80e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T123011865142Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39724cc3a062bf77992d0359f8051017880eba9d525a14b538de629ca2ffaf4f +size 6043177 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T123014829626Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T123014829626Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a07e7c00b17addfe8312edfca4149f68a54cb8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T123014829626Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 18169325568, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:14.829067Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 17664000, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 5, + "inactive_split_bytes_current": 6200591360, + "inactive_split_bytes_peak": 8109701120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:12.000308Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 3988516864, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 6594812928, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142233042944, + "reserved_bytes_current_delta": 29198647296, + "reserved_bytes_peak": 142233042944, + "reserved_bytes_peak_delta": 29198647296, + "segment_count_current": 694, + "segment_count_current_delta": 153, + "segment_count_peak": 694, + "segment_count_peak_delta": 153, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T123014829626Z.pickle", + "timestamp_utc": "2026-08-01T12:30:14.955736Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T123014829626Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T123014829626Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a816ac2690cbcbd63488898f96b0a4ff929f240e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T123014829626Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19bc15821729abcc2e4f10f824748a666b7c13aa6c344060ffdc2048d5a38def +size 6183776 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T123016026596Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T123016026596Z.json new file mode 100644 index 0000000000000000000000000000000000000000..988b2dd00a44f0657b423bc781c311edf90e1907 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T123016026596Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:16.026104Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:14.976754Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142233042944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142233042944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 694, + "segment_count_current_delta": 0, + "segment_count_peak": 694, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T123016026596Z.pickle", + "timestamp_utc": "2026-08-01T12:30:16.032141Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T123016026596Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T123016026596Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d1fc8877d38ad06e9c750dad8c28b1337fb80bee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T123016026596Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f945faaab223ac16112f19bae64284941857a509ceda1e5b2db8fcbf03998be +size 177967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T123017082317Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T123017082317Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9649011daf30a1f81ad862c26c4fa581591857ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T123017082317Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:17.081816Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:16.033071Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142233042944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142233042944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 694, + "segment_count_current_delta": 0, + "segment_count_peak": 694, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T123017082317Z.pickle", + "timestamp_utc": "2026-08-01T12:30:17.088165Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T123017082317Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T123017082317Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d35df813e7750b031997c8a7d443333b9e0453c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T123017082317Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d9e100d7231f446255993653ccc220744bb74c5d4972d405fb3d93210d5c27d +size 177967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T123018137353Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T123018137353Z.json new file mode 100644 index 0000000000000000000000000000000000000000..660947577294b761ad341bb7accce531af89eaac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T123018137353Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:18.136949Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:17.089182Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142233042944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142233042944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 694, + "segment_count_current_delta": 0, + "segment_count_peak": 694, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T123018137353Z.pickle", + "timestamp_utc": "2026-08-01T12:30:18.142250Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T123018137353Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T123018137353Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80cf5ce5c045944afa83a952f982bc3ced2496f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T123018137353Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbb798d5fedccc25e284716b25b9bb347dcbe6ee58545ea19a7d4e28ba023a1b +size 177967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T123019189088Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T123019189088Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4be7ca7c4d8ff3e56207904dd48cb6fca8b8b837 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T123019189088Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:19.188658Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:18.143120Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142233042944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142233042944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 694, + "segment_count_current_delta": 0, + "segment_count_peak": 694, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T123019189088Z.pickle", + "timestamp_utc": "2026-08-01T12:30:19.194141Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T123019189088Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T123019189088Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3fe08c954764e490b809c84117c0a26ca7bcbb65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T123019189088Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbbff0777843a0cb61ffc6f792bb70383597706e8da8bc46432d6314b3ebc079 +size 177967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T123020240996Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T123020240996Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f476d37bb794b34ee4ea1904be9521ecafc130e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T123020240996Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:20.240561Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:19.195068Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142233042944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142233042944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 694, + "segment_count_current_delta": 0, + "segment_count_peak": 694, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T123020240996Z.pickle", + "timestamp_utc": "2026-08-01T12:30:20.246114Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T123020240996Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T123020240996Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6685da0151fa86f9c35cd8ff139568e9efcfeeb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T123020240996Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04af0fa82abf14b30710a992f1b53b4f78c29f60cd7db50c5dc06364a989133b +size 177967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T123122076635Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T123122076635Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94e90a51da06f624aac4ed9b4bda38ad16d60dab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T123122076635Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61317944320, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61317944320, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 61317944320, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 6076132352, + "inactive_split_bytes_peak": 14704514048, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 142302248960, + "reserved_bytes_peak": 142302248960, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-08-01T12:31:22.076150Z", + "world_size": 8 + }, + "allocated_bytes_current": 61317944320, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 5, + "inactive_split_bytes_current": 10189108224, + "inactive_split_bytes_peak": 14704514048, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:20.247050Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6076132352, + "inactive_split_bytes_current_delta": -4112975872, + "inactive_split_bytes_peak": 14704514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 142302248960, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 142302248960, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 699, + "segment_count_current_delta": 5, + "segment_count_peak": 699, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T123122076635Z.pickle", + "timestamp_utc": "2026-08-01T12:31:22.164644Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T123122076635Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T123122076635Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..15a246d402419e8c00e0cce444d02dbd71ce5173 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T123122076635Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ab925112bff0be2e0946952a0cb50d310aa49df3b3a31bb99aabd79e9fadc02 +size 3840983 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T123009322767Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T123009322767Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4bb42f59d3394e5c5ad75ef36173c6faba12cd03 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T123009322767Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 87042705408, + "active_bytes_peak_delta": 51576369152, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87042705408, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87042705408, + "device": 6, + "inactive_split_bytes_current": 4845831168, + "inactive_split_bytes_peak": 8075752448, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 95711920128, + "reserved_bytes_peak": 95711920128, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T12:30:09.322143Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 87042705408, + "allocated_bytes_peak_delta": 51576369152, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:29:00.740490Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4845831168, + "inactive_split_bytes_current_delta": 4713012224, + "inactive_split_bytes_peak": 8075752448, + "inactive_split_bytes_peak_delta": 7915366400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 95711920128, + "reserved_bytes_current_delta": 56467914752, + "reserved_bytes_peak": 95711920128, + "reserved_bytes_peak_delta": 56467914752, + "segment_count_current": 461, + "segment_count_current_delta": 390, + "segment_count_peak": 461, + "segment_count_peak_delta": 390, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T123009322767Z.pickle", + "timestamp_utc": "2026-08-01T12:30:09.441543Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T123009322767Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T123009322767Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9aff614b0b7b8018648b8e82b38af1231320a2ef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T123009322767Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d89dd6338dd8799e8dfa885e6a686449090965319a65f5c307c086499f4e484 +size 5940616 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T123010630945Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T123010630945Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a6cce3583ff09bc45cd59a43a4d8529e64d16bf2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T123010630945Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110683463680, + "active_bytes_peak_delta": 23640758272, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 6, + "inactive_split_bytes_current": 6433375232, + "inactive_split_bytes_peak": 8109701120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.630528Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683463680, + "allocated_bytes_peak_delta": 23640758272, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87042705408, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87042705408, + "device": 6, + "inactive_split_bytes_current": 4845831168, + "inactive_split_bytes_peak": 8075752448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 95711920128, + "reserved_bytes_peak": 95711920128, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T12:30:09.460368Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6433375232, + "inactive_split_bytes_current_delta": 1587544064, + "inactive_split_bytes_peak": 8109701120, + "inactive_split_bytes_peak_delta": 33948672, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113034395648, + "reserved_bytes_current_delta": 17322475520, + "reserved_bytes_peak": 113034395648, + "reserved_bytes_peak_delta": 17322475520, + "segment_count_current": 541, + "segment_count_current_delta": 80, + "segment_count_peak": 541, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T123010630945Z.pickle", + "timestamp_utc": "2026-08-01T12:30:10.740546Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T123010630945Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T123010630945Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94cdffb21c1972fbc2a1da3e92e0a8b7c148a58a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T123010630945Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:545a5c766d1b9ae0b4a860a0607496fc8b7b0a06885a69b7a809e8c8fa411bcb +size 5922783 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T123011865177Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T123011865177Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1cf17156844b44449861c5ad6bf64072611af4d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T123011865177Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110683463680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 6, + "inactive_split_bytes_current": 6200591360, + "inactive_split_bytes_peak": 8109701120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:11.864776Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683463680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 6, + "inactive_split_bytes_current": 6433375232, + "inactive_split_bytes_peak": 8109701120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.754664Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6200591360, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 8109701120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113034395648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113034395648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 541, + "segment_count_current_delta": 0, + "segment_count_peak": 541, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T123011865177Z.pickle", + "timestamp_utc": "2026-08-01T12:30:11.972619Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T123011865177Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T123011865177Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..611350bd3e3d17bb7d724fc50b104c256ab1dc24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T123011865177Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beeb837e8d5c47da50e82e8736dd888263a8918ded9dfb1e3b7463eee89b8150 +size 6574981 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T123014829874Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T123014829874Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6114e6be621c820eeebf6c475bc8c7a03ce1b499 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T123014829874Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 18169325568, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_peak": 14702416896, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:14.829310Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 17664000, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 110683463680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110683463680, + "device": 6, + "inactive_split_bytes_current": 6200591360, + "inactive_split_bytes_peak": 8109701120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:11.986061Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_current_delta": 3986419712, + "inactive_split_bytes_peak": 14702416896, + "inactive_split_bytes_peak_delta": 6592715776, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142233042944, + "reserved_bytes_current_delta": 29198647296, + "reserved_bytes_peak": 142233042944, + "reserved_bytes_peak_delta": 29198647296, + "segment_count_current": 694, + "segment_count_current_delta": 153, + "segment_count_peak": 694, + "segment_count_peak_delta": 153, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T123014829874Z.pickle", + "timestamp_utc": "2026-08-01T12:30:14.958286Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T123014829874Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T123014829874Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c17a266f008029cd599c0a393b6708530976c8d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T123014829874Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f3aa8e51083bb9c8cff5b1338037f972fcd870f05262967a36431beb51727d5 +size 6183956 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T123016026592Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T123016026592Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d74ee6e4ef1d01fbe152c67a89565253e7123bb3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T123016026592Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_peak": 14702416896, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:16.026119Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_peak": 14702416896, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:14.976165Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14702416896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142233042944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142233042944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 694, + "segment_count_current_delta": 0, + "segment_count_peak": 694, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T123016026592Z.pickle", + "timestamp_utc": "2026-08-01T12:30:16.031965Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T123016026592Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T123016026592Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f03be1b74a7b0f59204bb77a3eceb6e80e02073 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T123016026592Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:117ea5dd60b2a456f1d1650be54e95f4dbcef764c86510dd3a3ba3a2119d3306 +size 177967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T123017082388Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T123017082388Z.json new file mode 100644 index 0000000000000000000000000000000000000000..541fe27790f31b2975722404a608ff259d89727f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T123017082388Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_peak": 14702416896, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:17.081862Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_peak": 14702416896, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:16.032890Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14702416896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142233042944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142233042944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 694, + "segment_count_current_delta": 0, + "segment_count_peak": 694, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T123017082388Z.pickle", + "timestamp_utc": "2026-08-01T12:30:17.088734Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T123017082388Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T123017082388Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..852fe939bb7b2e8fac160088f8ee409c241f820e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T123017082388Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa44e92667fd3b1a53eadda0b1cd8acafe67fa62dd77524a00852eeae232558f +size 177967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T123018137435Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T123018137435Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eecfd39fa8a14b0747b420f763b7476bb4e62490 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T123018137435Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_peak": 14702416896, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:18.136996Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_peak": 14702416896, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:17.089711Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14702416896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142233042944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142233042944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 694, + "segment_count_current_delta": 0, + "segment_count_peak": 694, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T123018137435Z.pickle", + "timestamp_utc": "2026-08-01T12:30:18.142710Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T123018137435Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T123018137435Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b07e2dbee8fe44a3a62e0df5739c5fc73b9270fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T123018137435Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0706977fc51d73da17a0c2e677e6736c70955f4d44ada3659c8a4d2497ff5d6 +size 177967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T123019189091Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T123019189091Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bd1443a4e17f01d5979829f687ecf33730169edb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T123019189091Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_peak": 14702416896, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:19.188667Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_peak": 14702416896, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:18.143604Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14702416896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142233042944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142233042944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 694, + "segment_count_current_delta": 0, + "segment_count_peak": 694, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T123019189091Z.pickle", + "timestamp_utc": "2026-08-01T12:30:19.194289Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T123019189091Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T123019189091Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3aeae032b8c776bc9603e52d8def90980a826fde --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T123019189091Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4d025532bc5c8078b54580573f2308147ad2bf6f8a643d6e992455ebb66a5db +size 177967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T123020241006Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T123020241006Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2108e1ba6ebc94e8f2e08b6d6a94403806ec159a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T123020241006Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_peak": 14702416896, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:20.240581Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_peak": 14702416896, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:19.195222Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14702416896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142233042944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142233042944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 694, + "segment_count_current_delta": 0, + "segment_count_peak": 694, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T123020241006Z.pickle", + "timestamp_utc": "2026-08-01T12:30:20.246271Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T123020241006Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T123020241006Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ad5d1b4b5d2bb76061f81448ed5c2c3cedba661 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T123020241006Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad663df0593c9bf5b70194c0aec9e8819568baa5c47df89078c7fc45b94767dc +size 177967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T123122076620Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T123122076620Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5f99676f2dcccc20eadcb0b31b2301d5352de76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T123122076620Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61317944320, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 128852789248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61317944320, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 61317944320, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 6074035200, + "inactive_split_bytes_peak": 14702416896, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 142302248960, + "reserved_bytes_peak": 142302248960, + "segment_count_current": 699, + "segment_count_peak": 699, + "timestamp_utc": "2026-08-01T12:31:22.076126Z", + "world_size": 8 + }, + "allocated_bytes_current": 61317944320, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 110701127680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 128852789248, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 110701127680, + "device": 6, + "inactive_split_bytes_current": 10187011072, + "inactive_split_bytes_peak": 14702416896, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 142233042944, + "reserved_bytes_peak": 142233042944, + "segment_count_current": 694, + "segment_count_peak": 694, + "timestamp_utc": "2026-08-01T12:30:20.247155Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6074035200, + "inactive_split_bytes_current_delta": -4112975872, + "inactive_split_bytes_peak": 14702416896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 142302248960, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 142302248960, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 699, + "segment_count_current_delta": 5, + "segment_count_peak": 699, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T123122076620Z.pickle", + "timestamp_utc": "2026-08-01T12:31:22.156189Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T123122076620Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T123122076620Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2f02a7d49f985e14f437847a127fc5b26cd680f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T123122076620Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a0438d52f62e1b944514174811bdd64779bd6fe1a9f77d30d8b5d31ded7df5c +size 3841288 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T123009322557Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T123009322557Z.json new file mode 100644 index 0000000000000000000000000000000000000000..468e16e73eb6b60b2e776c5511a16bc3eef06c6a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T123009322557Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 37276165632, + "active_bytes_peak": 87042705408, + "active_bytes_peak_delta": 51576369152, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 87042705408, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 87042705408, + "device": 7, + "inactive_split_bytes_current": 4843733504, + "inactive_split_bytes_peak": 8077849600, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 95711920128, + "reserved_bytes_peak": 95711920128, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T12:30:09.321980Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 37276165632, + "allocated_bytes_peak": 87042705408, + "allocated_bytes_peak_delta": 51576369152, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:29:00.740449Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4843733504, + "inactive_split_bytes_current_delta": 4710914560, + "inactive_split_bytes_peak": 8077849600, + "inactive_split_bytes_peak_delta": 7917463552, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 95711920128, + "reserved_bytes_current_delta": 56467914752, + "reserved_bytes_peak": 95711920128, + "reserved_bytes_peak_delta": 56467914752, + "segment_count_current": 461, + "segment_count_current_delta": 390, + "segment_count_peak": 461, + "segment_count_peak_delta": 390, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T123009322557Z.pickle", + "timestamp_utc": "2026-08-01T12:30:09.438577Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T123009322557Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T123009322557Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..882f967a6f4f8fa43c888c0ff7f304239d7a7aba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T123009322557Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8e71bae8b01bc7bf3972aad197ca6fb8262059b3b7f311ea4ce1edc8fa9f8db +size 5942204 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T123010630977Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T123010630977Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fcd783a73327c17664b352d4327fad2f77507ff2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T123010630977Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110683464192, + "active_bytes_peak_delta": 23640758784, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 110683464192, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 110683464192, + "device": 7, + "inactive_split_bytes_current": 6437569024, + "inactive_split_bytes_peak": 8105506304, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.630570Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683464192, + "allocated_bytes_peak_delta": 23640758784, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 87042705408, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 87042705408, + "device": 7, + "inactive_split_bytes_current": 4843733504, + "inactive_split_bytes_peak": 8077849600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 95711920128, + "reserved_bytes_peak": 95711920128, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T12:30:09.457390Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6437569024, + "inactive_split_bytes_current_delta": 1593835520, + "inactive_split_bytes_peak": 8105506304, + "inactive_split_bytes_peak_delta": 27656704, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113034395648, + "reserved_bytes_current_delta": 17322475520, + "reserved_bytes_peak": 113034395648, + "reserved_bytes_peak_delta": 17322475520, + "segment_count_current": 541, + "segment_count_current_delta": 80, + "segment_count_peak": 541, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T123010630977Z.pickle", + "timestamp_utc": "2026-08-01T12:30:10.746351Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T123010630977Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T123010630977Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e9a5b58fd3bcdf61872ec1a6c448ca6d8abb100 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T123010630977Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:267b6d5536b087c8026ee966e87511a8c110f8bb9cfa156a9b6e14b304d4a8d5 +size 5919348 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T123011865203Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T123011865203Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e2c523d54313bc640e57a5f3c638e8b997bc267b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T123011865203Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110683464192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 110683464192, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 110683464192, + "device": 7, + "inactive_split_bytes_current": 6198493696, + "inactive_split_bytes_peak": 8105506304, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:11.864788Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683464192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 110683464192, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 110683464192, + "device": 7, + "inactive_split_bytes_current": 6437569024, + "inactive_split_bytes_peak": 8105506304, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:10.762696Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6198493696, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 8105506304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113034395648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113034395648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 541, + "segment_count_current_delta": 0, + "segment_count_peak": 541, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T123011865203Z.pickle", + "timestamp_utc": "2026-08-01T12:30:11.975734Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T123011865203Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T123011865203Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0d1539f6eb089d846ec9f1d8f643c5fe05bdd401 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T123011865203Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91d6024591c3d4700e88e35dec8271cfc20532bbde7f7b6645edd7872d9a2560 +size 6044962 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T123014829647Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T123014829647Z.json new file mode 100644 index 0000000000000000000000000000000000000000..68ed2a857699bf67fe8a66fb05c4a0e3025d9c50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T123014829647Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524664320, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 128852789760, + "active_bytes_peak_delta": 18169325568, + "after": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_peak": 14702416384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:14.829112Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524664320, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110701128192, + "allocated_bytes_peak_delta": 17664000, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 110683464192, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 110683464192, + "device": 7, + "inactive_split_bytes_current": 6198493696, + "inactive_split_bytes_peak": 8105506304, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113034395648, + "reserved_bytes_peak": 113034395648, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:30:11.991360Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_current_delta": 3988516864, + "inactive_split_bytes_peak": 14702416384, + "inactive_split_bytes_peak_delta": 6596910080, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 29196550144, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 29196550144, + "segment_count_current": 693, + "segment_count_current_delta": 152, + "segment_count_peak": 693, + "segment_count_peak_delta": 152, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T123014829647Z.pickle", + "timestamp_utc": "2026-08-01T12:30:14.957314Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T123014829647Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T123014829647Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c1e60828a885d57a8c755d90cba6a36ac5a98713 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T123014829647Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c64aeabf7c07d651a35202696099e6fbdc8b58b152172721e1976db2fae0eb82 +size 6185478 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T123016026648Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T123016026648Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9b545c601bb11a99a44e5a7cf13837a72e4d005d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T123016026648Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524664320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_peak": 14702416384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:16.026153Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524664320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701128192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_peak": 14702416384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:14.974589Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14702416384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T123016026648Z.pickle", + "timestamp_utc": "2026-08-01T12:30:16.032091Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T123016026648Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T123016026648Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ae9d0403f96721d11c18ea2cfb8096cbc1f769d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T123016026648Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d2b26785feb73e887ea895d187e46e9e9cf0a7f593ac247fec21ca16880be66 +size 179617 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T123017082371Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T123017082371Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2820fb76a6e3fd7abad1c311b8c1c456f30e9463 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T123017082371Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524664320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_peak": 14702416384, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:17.081871Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524664320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701128192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_peak": 14702416384, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:16.033069Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14702416384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T123017082371Z.pickle", + "timestamp_utc": "2026-08-01T12:30:17.088207Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T123017082371Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T123017082371Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e492f8775fba5227a112c4fb931870065e6e8b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T123017082371Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:505a91bc3f9e7dddb40476ebfa0d806d8247ebef199494c48973626dcd6821de +size 179617 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T123018137407Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T123018137407Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d2d3904ad45680241a719c691d9f79a0e469feaf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T123018137407Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524664320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_peak": 14702416384, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:18.136989Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524664320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701128192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_peak": 14702416384, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:17.089103Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14702416384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T123018137407Z.pickle", + "timestamp_utc": "2026-08-01T12:30:18.142267Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T123018137407Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T123018137407Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d69c5b50378de0d65641671e2672e2eab26d89cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T123018137407Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9c6d88280b89c6744ee485ec5b3b961d4f17988b4cc4ac24f2f378a48a3d4dc +size 179617 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T123019189121Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T123019189121Z.json new file mode 100644 index 0000000000000000000000000000000000000000..024aa6610feb4f1113a7dd9a2579ba04a4df5cbf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T123019189121Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524664320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_peak": 14702416384, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:19.188699Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524664320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701128192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_peak": 14702416384, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:18.143125Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14702416384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T123019189121Z.pickle", + "timestamp_utc": "2026-08-01T12:30:19.194048Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T123019189121Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T123019189121Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5793ca8a5940d1b18523419577e3c8624ccd6be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T123019189121Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7d77ed94606cfc84dc2dc3ef5571a51d78911e75e32a901c6748755329650a5 +size 179617 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T123020241032Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T123020241032Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b68f973b4e4bc37e5ebf5d07e0f15b5d941b89b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T123020241032Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524664320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 128852789760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_peak": 14702416384, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:20.240608Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524664320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110701128192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_peak": 14702416384, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:19.194936Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14702416384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142230945792, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 142230945792, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 693, + "segment_count_current_delta": 0, + "segment_count_peak": 693, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T123020241032Z.pickle", + "timestamp_utc": "2026-08-01T12:30:20.246154Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T123020241032Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T123020241032Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6690e9db9b635dcb389b9609cf1763edd1a5dc2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T123020241032Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d799aa2f1eefbe10f35e008aa36767441356cd31cfbe0a7244fd2500fd9047fc +size 179617 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T123122076647Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T123122076647Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6b6cc755a6f7c5c9bb3d3ead5c88bdec67750531 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T123122076647Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61317944832, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 128852789760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61317944832, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 61317944832, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 6074034688, + "inactive_split_bytes_peak": 14702416384, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 142300151808, + "reserved_bytes_peak": 142300151808, + "segment_count_current": 698, + "segment_count_peak": 698, + "timestamp_utc": "2026-08-01T12:31:22.076171Z", + "world_size": 8 + }, + "allocated_bytes_current": 61317944832, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 110701128192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 128852789760, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 110701128192, + "device": 7, + "inactive_split_bytes_current": 10187010560, + "inactive_split_bytes_peak": 14702416384, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 142230945792, + "reserved_bytes_peak": 142230945792, + "segment_count_current": 693, + "segment_count_peak": 693, + "timestamp_utc": "2026-08-01T12:30:20.247028Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6074034688, + "inactive_split_bytes_current_delta": -4112975872, + "inactive_split_bytes_peak": 14702416384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 142300151808, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 142300151808, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 698, + "segment_count_current_delta": 5, + "segment_count_peak": 698, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T123122076647Z.pickle", + "timestamp_utc": "2026-08-01T12:31:22.160556Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T123122076647Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T123122076647Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b3f1cb557b862082eb358a058958aa2531d3dc20 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T123122076647Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d2f37213d0ea4bf3aa46cce8e3ab8d77905a2303b7634cb5ddd990495fa1172 +size 3842824 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..aeac69723a24d667313416b12cde5279c59709a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 4096 +capacity: 1.00 +expert_dispatch_overlap: on +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6014 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 241 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T123419487342Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T123419487342Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1da469da8e7374fe1e507860d0ffcbb6b7d3396d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T123419487342Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 87154351104, + "active_bytes_peak_delta": 51688014848, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87154351104, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87154351104, + "device": 0, + "inactive_split_bytes_current": 5564843008, + "inactive_split_bytes_peak": 7778160640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 96403980288, + "reserved_bytes_peak": 96403980288, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T12:34:19.486736Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 87154351104, + "allocated_bytes_peak_delta": 51688014848, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:33:11.535621Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5564843008, + "inactive_split_bytes_current_delta": 5429926912, + "inactive_split_bytes_peak": 7778160640, + "inactive_split_bytes_peak_delta": 7617528832, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 96403980288, + "reserved_bytes_current_delta": 57157877760, + "reserved_bytes_peak": 96403980288, + "reserved_bytes_peak_delta": 57157877760, + "segment_count_current": 501, + "segment_count_current_delta": 429, + "segment_count_peak": 501, + "segment_count_peak_delta": 429, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T123419487342Z.pickle", + "timestamp_utc": "2026-08-01T12:34:19.607232Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T123419487342Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T123419487342Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..41e6e238e95d279a2368c7b00e32467815c76a05 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T123419487342Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:981ee07d9f2b9c50215dcc8f5fd8ab2efb39c1c727e4f9202a816cacd5ea16a3 +size 5405002 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T123420636479Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T123420636479Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f73c79677f033d990f98082aacee60fc4b87b4a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T123420636479Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110783696384, + "active_bytes_peak_delta": 23629345280, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110783696384, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110783696384, + "device": 0, + "inactive_split_bytes_current": 5590008832, + "inactive_split_bytes_peak": 7778160640, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 114078777344, + "reserved_bytes_peak": 114078777344, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T12:34:20.636063Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110783696384, + "allocated_bytes_peak_delta": 23629345280, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87154351104, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87154351104, + "device": 0, + "inactive_split_bytes_current": 5564843008, + "inactive_split_bytes_peak": 7778160640, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 96403980288, + "reserved_bytes_peak": 96403980288, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T12:34:19.627499Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5590008832, + "inactive_split_bytes_current_delta": 25165824, + "inactive_split_bytes_peak": 7778160640, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 114078777344, + "reserved_bytes_current_delta": 17674797056, + "reserved_bytes_peak": 114078777344, + "reserved_bytes_peak_delta": 17674797056, + "segment_count_current": 573, + "segment_count_current_delta": 72, + "segment_count_peak": 573, + "segment_count_peak_delta": 72, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T123420636479Z.pickle", + "timestamp_utc": "2026-08-01T12:34:20.743995Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T123420636479Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T123420636479Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..49c6de02250447276e134cb50b076d007fa35183 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T123420636479Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ed3bc645fa341b94549f6751867f02ae83fb99e156e87ad2e2e615dfe2cb574 +size 5654249 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T123421749265Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T123421749265Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5195282c47b416da4383cba7860a919d3ff219f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T123421749265Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110783696384, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110783696384, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110783696384, + "device": 0, + "inactive_split_bytes_current": 5353030656, + "inactive_split_bytes_peak": 8091990016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 114783420416, + "reserved_bytes_peak": 114783420416, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T12:34:21.748861Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110783696384, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110783696384, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110783696384, + "device": 0, + "inactive_split_bytes_current": 5590008832, + "inactive_split_bytes_peak": 7778160640, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 114078777344, + "reserved_bytes_peak": 114078777344, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T12:34:20.759333Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5353030656, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 8091990016, + "inactive_split_bytes_peak_delta": 313829376, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 114783420416, + "reserved_bytes_current_delta": 704643072, + "reserved_bytes_peak": 114783420416, + "reserved_bytes_peak_delta": 704643072, + "segment_count_current": 576, + "segment_count_current_delta": 3, + "segment_count_peak": 576, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T123421749265Z.pickle", + "timestamp_utc": "2026-08-01T12:34:21.852740Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T123421749265Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T123421749265Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e6683e808d853308538d9a99711e1ed85019a2ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T123421749265Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b2b3c769357a12df90dcf6b83e04297a197c9ba94879ebe96e44107f584de3b +size 5592476 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T123424716352Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T123424716352Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9be2b20ee9f61637c0c82178892fc1e949cf9840 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T123424716352Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162774837248, + "active_bytes_peak_delta": 51991140864, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_peak": 12249988608, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_peak": 175598731264, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:34:24.715797Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110800308224, + "allocated_bytes_peak_delta": 16611840, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110783696384, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110783696384, + "device": 0, + "inactive_split_bytes_current": 5353030656, + "inactive_split_bytes_peak": 8091990016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 114783420416, + "reserved_bytes_peak": 114783420416, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T12:34:21.867453Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_current_delta": 322695168, + "inactive_split_bytes_peak": 12249988608, + "inactive_split_bytes_peak_delta": 4157998592, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_current_delta": 60815310848, + "reserved_bytes_peak": 175598731264, + "reserved_bytes_peak_delta": 60815310848, + "segment_count_current": 936, + "segment_count_current_delta": 360, + "segment_count_peak": 936, + "segment_count_peak_delta": 360, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T123424716352Z.pickle", + "timestamp_utc": "2026-08-01T12:34:24.836461Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T123424716352Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T123424716352Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f81bb7002f1c124b39bc4a8269f5442833ed5443 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T123424716352Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88e4b8ee750912f619ae4e990e86c68d0822750bf5a7173382e74b27eac8c46a +size 6620631 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T123425656585Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T123425656585Z.json new file mode 100644 index 0000000000000000000000000000000000000000..afd5f2fb792a4abe7c4060eb041bf4a0d24cb645 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T123425656585Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162774837248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_peak": 12249988608, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_peak": 175598731264, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:34:25.656115Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110800308224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_peak": 12249988608, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_peak": 175598731264, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:34:24.853252Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12249988608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175598731264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 936, + "segment_count_current_delta": 0, + "segment_count_peak": 936, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T123425656585Z.pickle", + "timestamp_utc": "2026-08-01T12:34:25.664130Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T123425656585Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T123425656585Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d28d7f70b554f6f9e2acc45cc455d5d2cfb1992 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T123425656585Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f6bc7bfd791e67c8d3da4a86d17e2c07cbcc27817afe75bc2936bf711f766d8 +size 274530 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T123426463881Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T123426463881Z.json new file mode 100644 index 0000000000000000000000000000000000000000..680951dba17e5b06dbb2d623c6a71db664c000b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T123426463881Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162774837248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_peak": 12249988608, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_peak": 175598731264, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:34:26.463469Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110800308224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_peak": 12249988608, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_peak": 175598731264, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:34:25.665117Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12249988608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175598731264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 936, + "segment_count_current_delta": 0, + "segment_count_peak": 936, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T123426463881Z.pickle", + "timestamp_utc": "2026-08-01T12:34:26.470859Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T123426463881Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T123426463881Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fa17d1971364cbf7d95bbdf932dc909afe34c5cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T123426463881Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14305509101b5a223161875f9e8b13b8f78ecaa107664cbaca8bfe29e924fd8d +size 274530 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T123427265204Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T123427265204Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b69d42368d11e6e45c1e1164d4c223c91314d8c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T123427265204Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162774837248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_peak": 12249988608, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_peak": 175598731264, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:34:27.264790Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110800308224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_peak": 12249988608, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_peak": 175598731264, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:34:26.471845Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12249988608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175598731264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 936, + "segment_count_current_delta": 0, + "segment_count_peak": 936, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T123427265204Z.pickle", + "timestamp_utc": "2026-08-01T12:34:27.271981Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T123427265204Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T123427265204Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2cf632123aa5a8c60fd5482fee918ddc04e0d542 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T123427265204Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a15f48ddd713c5e62df3363e7fb33de31fb31f1a9163aff2e266768e8815187 +size 274530 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T123428069035Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T123428069035Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f831c87e1fc583e663d28dc97be453b2145b8c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T123428069035Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162774837248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_peak": 12249988608, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_peak": 175598731264, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:34:28.068570Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110800308224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_peak": 12249988608, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_peak": 175598731264, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:34:27.272945Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12249988608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175598731264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 936, + "segment_count_current_delta": 0, + "segment_count_peak": 936, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T123428069035Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.076435Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T123428069035Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T123428069035Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ebc212438b746a0c81fa8a57014914df63afd7a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T123428069035Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e98c49195002e0c481bb7fefcdb2b9ca2eca285493b89b88648b4fccefacdf68 +size 274530 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T123428871879Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T123428871879Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6297f848b50a296f6616ac1f681918099967b8f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T123428871879Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162774837248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_peak": 12249988608, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_peak": 175598731264, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:34:28.871445Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110800308224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_peak": 12249988608, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_peak": 175598731264, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:34:28.077468Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12249988608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175598731264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 936, + "segment_count_current_delta": 0, + "segment_count_peak": 936, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T123428871879Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.879039Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T123428871879Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T123428871879Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ec51cbce158864c4209536f03df38d7cd6d4a627 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T123428871879Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3eca9ba4e56d51abed3a4dbf79185d3e393b1f7a3275bcd3d58f89dd071a2cec +size 274530 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T123529587108Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T123529587108Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f6155288880b7dcf81a675d6cabf3e5a96e79a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T123529587108Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 162774837248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5413432320, + "inactive_split_bytes_peak": 12249988608, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 177431642112, + "reserved_bytes_peak": 177431642112, + "segment_count_current": 945, + "segment_count_peak": 945, + "timestamp_utc": "2026-08-01T12:35:29.586650Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110800308224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162774837248, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110800308224, + "device": 0, + "inactive_split_bytes_current": 5675725824, + "inactive_split_bytes_peak": 12249988608, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 175598731264, + "reserved_bytes_peak": 175598731264, + "segment_count_current": 936, + "segment_count_peak": 936, + "timestamp_utc": "2026-08-01T12:34:28.880018Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5413432320, + "inactive_split_bytes_current_delta": -262293504, + "inactive_split_bytes_peak": 12249988608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 177431642112, + "reserved_bytes_current_delta": 1832910848, + "reserved_bytes_peak": 177431642112, + "reserved_bytes_peak_delta": 1832910848, + "segment_count_current": 945, + "segment_count_current_delta": 9, + "segment_count_peak": 945, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T123529587108Z.pickle", + "timestamp_utc": "2026-08-01T12:35:29.677169Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T123529587108Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T123529587108Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe4eaf849256af4c719e29d558f427549eb4bd20 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T123529587108Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:889472432fa11cd704a14cc6bb21b5165b8b6af945c3ae60502c75f9cd84f141 +size 4220272 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T123419487109Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T123419487109Z.json new file mode 100644 index 0000000000000000000000000000000000000000..da0165b5164d5c2c6b2f9153c5c9891df66a212e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T123419487109Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 87155254272, + "active_bytes_peak_delta": 51688918016, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87155254272, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87155254272, + "device": 1, + "inactive_split_bytes_current": 5736809472, + "inactive_split_bytes_peak": 8371820032, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 96636764160, + "reserved_bytes_peak": 96636764160, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T12:34:19.486493Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 87155254272, + "allocated_bytes_peak_delta": 51688918016, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:33:11.535571Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5736809472, + "inactive_split_bytes_current_delta": 5603990528, + "inactive_split_bytes_peak": 8371820032, + "inactive_split_bytes_peak_delta": 8211433984, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 96636764160, + "reserved_bytes_current_delta": 57392758784, + "reserved_bytes_peak": 96636764160, + "reserved_bytes_peak_delta": 57392758784, + "segment_count_current": 501, + "segment_count_current_delta": 430, + "segment_count_peak": 501, + "segment_count_peak_delta": 430, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T123419487109Z.pickle", + "timestamp_utc": "2026-08-01T12:34:19.602711Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T123419487109Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T123419487109Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..84cc17462a9ebcdcb93752937291d46f574c04c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T123419487109Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8acf3e8bacf30119f08581018c043b0cf18bcf3b9ccd88c0d14217d77471d4f9 +size 5396168 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T123420636360Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T123420636360Z.json new file mode 100644 index 0000000000000000000000000000000000000000..810aabf7703b28da10552e6a8e56ba35571a48d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T123420636360Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110774482944, + "active_bytes_peak_delta": 23619228672, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110774482944, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110774482944, + "device": 1, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_peak": 8371820032, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 114240258048, + "reserved_bytes_peak": 114240258048, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T12:34:20.635942Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110774482944, + "allocated_bytes_peak_delta": 23619228672, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87155254272, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87155254272, + "device": 1, + "inactive_split_bytes_current": 5736809472, + "inactive_split_bytes_peak": 8371820032, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 96636764160, + "reserved_bytes_peak": 96636764160, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T12:34:19.619185Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_current_delta": -446693376, + "inactive_split_bytes_peak": 8371820032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 114240258048, + "reserved_bytes_current_delta": 17603493888, + "reserved_bytes_peak": 114240258048, + "reserved_bytes_peak_delta": 17603493888, + "segment_count_current": 570, + "segment_count_current_delta": 69, + "segment_count_peak": 570, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T123420636360Z.pickle", + "timestamp_utc": "2026-08-01T12:34:20.736479Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T123420636360Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T123420636360Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..89d75e5b5d85e3ce43f282ede12d6bd06b5f49b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T123420636360Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bcb836a110e411e448682f0148cf7a3e352059b622902866e49cd5bce6abf48 +size 5650502 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T123421749170Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T123421749170Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a55d42fb374ad2c25d5c0425ffc6e21da353d4f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T123421749170Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110774482944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110774482944, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110774482944, + "device": 1, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_peak": 8371820032, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115769081856, + "reserved_bytes_peak": 115769081856, + "segment_count_current": 574, + "segment_count_peak": 574, + "timestamp_utc": "2026-08-01T12:34:21.748754Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110774482944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110774482944, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110774482944, + "device": 1, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_peak": 8371820032, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 114240258048, + "reserved_bytes_peak": 114240258048, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T12:34:20.748249Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8371820032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115769081856, + "reserved_bytes_current_delta": 1528823808, + "reserved_bytes_peak": 115769081856, + "reserved_bytes_peak_delta": 1528823808, + "segment_count_current": 574, + "segment_count_current_delta": 4, + "segment_count_peak": 574, + "segment_count_peak_delta": 4, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T123421749170Z.pickle", + "timestamp_utc": "2026-08-01T12:34:21.846939Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T123421749170Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T123421749170Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb1ad928cebeea31dbfc92dd54307f957f344180 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T123421749170Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e653d7ef5bb2542ebd507af9070d6602e63dd0119d7f50441ce3399594a892bb +size 5590628 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T123424716360Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T123424716360Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4459ffb60dbb51ac49f12f57ca5d4169a969686f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T123424716360Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 51986460672, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11601373184, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 175514845184, + "reserved_bytes_peak": 175514845184, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:24.715789Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 13241856, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110774482944, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110774482944, + "device": 1, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_peak": 8371820032, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 115769081856, + "reserved_bytes_peak": 115769081856, + "segment_count_current": 574, + "segment_count_peak": 574, + "timestamp_utc": "2026-08-01T12:34:21.858142Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 320598016, + "inactive_split_bytes_peak": 11601373184, + "inactive_split_bytes_peak_delta": 3229553152, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 175514845184, + "reserved_bytes_current_delta": 59745763328, + "reserved_bytes_peak": 175514845184, + "reserved_bytes_peak_delta": 59745763328, + "segment_count_current": 934, + "segment_count_current_delta": 360, + "segment_count_peak": 934, + "segment_count_peak_delta": 360, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T123424716360Z.pickle", + "timestamp_utc": "2026-08-01T12:34:24.831563Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T123424716360Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T123424716360Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c5bc2136035f89e1fda3ffe162b7e62cd2bc4ee1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T123424716360Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a061e44159e4e72521a6a1995c4041c0b671c05921afe6641e4baa38c7e6419 +size 5870653 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T123425656428Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T123425656428Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1acd0d74d65498e5b0bb9643eede9fb17ba9db7b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T123425656428Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11601373184, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 175514845184, + "reserved_bytes_peak": 175514845184, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:25.655979Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11601373184, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 175514845184, + "reserved_bytes_peak": 175514845184, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:24.845062Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11601373184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 175514845184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175514845184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T123425656428Z.pickle", + "timestamp_utc": "2026-08-01T12:34:25.662852Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T123425656428Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T123425656428Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9eea822c4017cff58cb32254278ef88f281531da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T123425656428Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b91d5f5c871451ce59ac80a4d4249ab5955a5f88f294fb5d2146f0a8ebec361f +size 240991 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T123426463781Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T123426463781Z.json new file mode 100644 index 0000000000000000000000000000000000000000..563f8bb042d9a7002c14842ed1d116d6635d834c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T123426463781Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11601373184, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 175514845184, + "reserved_bytes_peak": 175514845184, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:26.463371Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11601373184, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 175514845184, + "reserved_bytes_peak": 175514845184, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:25.663842Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11601373184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 175514845184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175514845184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T123426463781Z.pickle", + "timestamp_utc": "2026-08-01T12:34:26.470242Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T123426463781Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T123426463781Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..af1b8435b21a3d66723d4006dab2d936b30b3658 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T123426463781Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be692124dcec728d95bb85459a064036d3fe014036287b2a733a7cf5a13d2efd +size 240960 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T123427265078Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T123427265078Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7c66e39508ce51368e29bb18f00f1e82a9d47ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T123427265078Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11601373184, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 175514845184, + "reserved_bytes_peak": 175514845184, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:27.264656Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11601373184, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 175514845184, + "reserved_bytes_peak": 175514845184, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:26.471202Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11601373184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 175514845184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175514845184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T123427265078Z.pickle", + "timestamp_utc": "2026-08-01T12:34:27.271475Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T123427265078Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T123427265078Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..79b0109ea5ab7010236aac00906384bbb1abadf9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T123427265078Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0b962eb868747943eade75e423251182e7ae032c174c5679ef9b2d9773ff27a +size 240991 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T123428068818Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T123428068818Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18864cbe85f04b085369c8ab2afcc5283085188d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T123428068818Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11601373184, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 175514845184, + "reserved_bytes_peak": 175514845184, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:28.068387Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11601373184, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 175514845184, + "reserved_bytes_peak": 175514845184, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:27.272446Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11601373184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 175514845184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175514845184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T123428068818Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.075349Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T123428068818Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T123428068818Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c84b86cc867a1d54e7bed16de1445834bcbe0ee5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T123428068818Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6d956fba66111c7fd93bf82447290000fab7590263608c149681d1a314c11f9 +size 240960 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T123428871689Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T123428871689Z.json new file mode 100644 index 0000000000000000000000000000000000000000..11f474c0a4ffc91996e5747fc7c51e8021272c70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T123428871689Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11601373184, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 175514845184, + "reserved_bytes_peak": 175514845184, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:28.871287Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11601373184, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 175514845184, + "reserved_bytes_peak": 175514845184, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:28.076331Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11601373184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 175514845184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175514845184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T123428871689Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.878228Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T123428871689Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T123428871689Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9cbd3c1a29f7579fd7e965f2070b07e8ac10efba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T123428871689Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d72c22ee66eaf0fbcbf78bdcdbab78a7445eebc4397a6c573af53dc6e7370ee5 +size 240991 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T123529587081Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T123529587081Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18ddef51037c79fedd5c426a1a0410d27834f1a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T123529587081Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5348420608, + "inactive_split_bytes_peak": 11601373184, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 178522161152, + "reserved_bytes_peak": 178522161152, + "segment_count_current": 948, + "segment_count_peak": 948, + "timestamp_utc": "2026-08-01T12:35:29.586608Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 1, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11601373184, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 175514845184, + "reserved_bytes_peak": 175514845184, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:28.879185Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5348420608, + "inactive_split_bytes_current_delta": -262293504, + "inactive_split_bytes_peak": 11601373184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 178522161152, + "reserved_bytes_current_delta": 3007315968, + "reserved_bytes_peak": 178522161152, + "reserved_bytes_peak_delta": 3007315968, + "segment_count_current": 948, + "segment_count_current_delta": 14, + "segment_count_peak": 948, + "segment_count_peak_delta": 14, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T123529587081Z.pickle", + "timestamp_utc": "2026-08-01T12:35:29.675348Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T123529587081Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T123529587081Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a1b609a2f5caa7690097e34b08eae677a050a7c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T123529587081Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03b93fa9aa3ba617f71e49d1da4dbbcbb20db7dec158dccb39af724492f37496 +size 4080206 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T123419486988Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T123419486988Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31fb3eecbec511d71834da001bc8c9c64bfba251 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T123419486988Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 87155254272, + "active_bytes_peak_delta": 51688918016, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87155254272, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87155254272, + "device": 2, + "inactive_split_bytes_current": 5736809472, + "inactive_split_bytes_peak": 8369722880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 97106526208, + "reserved_bytes_peak": 97106526208, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:34:19.486356Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 87155254272, + "allocated_bytes_peak_delta": 51688918016, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:33:11.536170Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5736809472, + "inactive_split_bytes_current_delta": 5603990528, + "inactive_split_bytes_peak": 8369722880, + "inactive_split_bytes_peak_delta": 8209336832, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 97106526208, + "reserved_bytes_current_delta": 57862520832, + "reserved_bytes_peak": 97106526208, + "reserved_bytes_peak_delta": 57862520832, + "segment_count_current": 503, + "segment_count_current_delta": 432, + "segment_count_peak": 503, + "segment_count_peak_delta": 432, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T123419486988Z.pickle", + "timestamp_utc": "2026-08-01T12:34:19.599360Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T123419486988Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T123419486988Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fa1548fe0f7dbd8601eeff894bea0255c24d56a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T123419486988Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8232932f708c40b1a822f0f99f92d85e380ce49b9d79885a0def407202ed3db +size 5396543 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T123420636351Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T123420636351Z.json new file mode 100644 index 0000000000000000000000000000000000000000..402f5be171985e7cc7e41f16215243bc687b392e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T123420636351Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110776034304, + "active_bytes_peak_delta": 23620780032, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110776034304, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110776034304, + "device": 2, + "inactive_split_bytes_current": 5288018944, + "inactive_split_bytes_peak": 8369722880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 114307366912, + "reserved_bytes_peak": 114307366912, + "segment_count_current": 571, + "segment_count_peak": 571, + "timestamp_utc": "2026-08-01T12:34:20.635929Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110776034304, + "allocated_bytes_peak_delta": 23620780032, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87155254272, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87155254272, + "device": 2, + "inactive_split_bytes_current": 5736809472, + "inactive_split_bytes_peak": 8369722880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 97106526208, + "reserved_bytes_peak": 97106526208, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:34:19.618933Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5288018944, + "inactive_split_bytes_current_delta": -448790528, + "inactive_split_bytes_peak": 8369722880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 114307366912, + "reserved_bytes_current_delta": 17200840704, + "reserved_bytes_peak": 114307366912, + "reserved_bytes_peak_delta": 17200840704, + "segment_count_current": 571, + "segment_count_current_delta": 68, + "segment_count_peak": 571, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T123420636351Z.pickle", + "timestamp_utc": "2026-08-01T12:34:20.741706Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T123420636351Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T123420636351Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f8bad0e4d2f93fbfb74442840143e4a840b00982 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T123420636351Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa94c74ef5389f2ad348537983ac694467ce8f33356243204d70d5e243c84c0c +size 5650457 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T123421749130Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T123421749130Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e4023eb827e84eed5ec904138fe043da8320061c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T123421749130Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110777165824, + "active_bytes_peak_delta": 1131520, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110777165824, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110777165824, + "device": 2, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_peak": 8369722880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 115012009984, + "reserved_bytes_peak": 115012009984, + "segment_count_current": 574, + "segment_count_peak": 574, + "timestamp_utc": "2026-08-01T12:34:21.748719Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110777165824, + "allocated_bytes_peak_delta": 1131520, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110776034304, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110776034304, + "device": 2, + "inactive_split_bytes_current": 5288018944, + "inactive_split_bytes_peak": 8369722880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 114307366912, + "reserved_bytes_peak": 114307366912, + "segment_count_current": 571, + "segment_count_peak": 571, + "timestamp_utc": "2026-08-01T12:34:20.756708Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 8369722880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 115012009984, + "reserved_bytes_current_delta": 704643072, + "reserved_bytes_peak": 115012009984, + "reserved_bytes_peak_delta": 704643072, + "segment_count_current": 574, + "segment_count_current_delta": 3, + "segment_count_peak": 574, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T123421749130Z.pickle", + "timestamp_utc": "2026-08-01T12:34:21.851751Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T123421749130Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T123421749130Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..476395747255c89cedfe02025955237b3e13cc9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T123421749130Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eab895b4b73a75b03520234b94439a018657c8fc28e21608b617d4345ab55317 +size 5590920 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T123424716184Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T123424716184Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae3e4fdd75e9118ef149c86a2bef51ea1c6a9097 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T123424716184Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162761992192, + "active_bytes_peak_delta": 51984826368, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11890911232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 175389016064, + "reserved_bytes_peak": 175389016064, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-01T12:34:24.715659Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110790608384, + "allocated_bytes_peak_delta": 13442560, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110777165824, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110777165824, + "device": 2, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_peak": 8369722880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 115012009984, + "reserved_bytes_peak": 115012009984, + "segment_count_current": 574, + "segment_count_peak": 574, + "timestamp_utc": "2026-08-01T12:34:21.866614Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 320598016, + "inactive_split_bytes_peak": 11890911232, + "inactive_split_bytes_peak_delta": 3521188352, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 175389016064, + "reserved_bytes_current_delta": 60377006080, + "reserved_bytes_peak": 175389016064, + "reserved_bytes_peak_delta": 60377006080, + "segment_count_current": 930, + "segment_count_current_delta": 356, + "segment_count_peak": 930, + "segment_count_peak_delta": 356, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T123424716184Z.pickle", + "timestamp_utc": "2026-08-01T12:34:24.835635Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T123424716184Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T123424716184Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..742a125ba435c589e1267e9af4f42c5ce812ef67 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T123424716184Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c03a9c84e12a64b6177fc50b38001dd95be1152410ab89e0f6e5ad2891075b45 +size 5870272 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T123425656427Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T123425656427Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7432626ccf3f05e7e4ffc6d2f0270ccacee902ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T123425656427Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162761992192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11890911232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 175389016064, + "reserved_bytes_peak": 175389016064, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-01T12:34:25.655967Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110790608384, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11890911232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 175389016064, + "reserved_bytes_peak": 175389016064, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-01T12:34:24.852393Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11890911232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 175389016064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175389016064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 930, + "segment_count_current_delta": 0, + "segment_count_peak": 930, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T123425656427Z.pickle", + "timestamp_utc": "2026-08-01T12:34:25.663583Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T123425656427Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T123425656427Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ed12d9cc01d181286747cb67b3c63a199feacc3a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T123425656427Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7324119a508951787d0a89db82c1d76eec3ab0188af5270886d88aa214e462c5 +size 241039 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T123426463751Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T123426463751Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f2b5849bb1aa4cac3e5c7a07fc1ebb64f313d31 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T123426463751Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162761992192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11890911232, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 175389016064, + "reserved_bytes_peak": 175389016064, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-01T12:34:26.463350Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110790608384, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11890911232, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 175389016064, + "reserved_bytes_peak": 175389016064, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-01T12:34:25.664611Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11890911232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 175389016064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175389016064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 930, + "segment_count_current_delta": 0, + "segment_count_peak": 930, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T123426463751Z.pickle", + "timestamp_utc": "2026-08-01T12:34:26.470396Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T123426463751Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T123426463751Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8c01cf294cc3a746ab5a877aa942de65422d8a1b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T123426463751Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33fec793e357d8b2ffada574bcbc1b7685650ba325be5849a5c57850a124f90b +size 241008 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T123427265013Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T123427265013Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0e4020ae3ac8afe08176fd4ffe5d0e50d2797543 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T123427265013Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162761992192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11890911232, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 175389016064, + "reserved_bytes_peak": 175389016064, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-01T12:34:27.264612Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110790608384, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11890911232, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 175389016064, + "reserved_bytes_peak": 175389016064, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-01T12:34:26.471351Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11890911232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 175389016064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175389016064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 930, + "segment_count_current_delta": 0, + "segment_count_peak": 930, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T123427265013Z.pickle", + "timestamp_utc": "2026-08-01T12:34:27.271494Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T123427265013Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T123427265013Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6d7ff0cec49182c89ae70b7357cd42f18ad3c7c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T123427265013Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:461d413cf2ed2c3ad44f383419284186f556b0165ee7378136556c5753ce96d7 +size 241039 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T123428068733Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T123428068733Z.json new file mode 100644 index 0000000000000000000000000000000000000000..444e08442091beaadcb6fc00361a0e3ae2dfd2c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T123428068733Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162761992192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11890911232, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 175389016064, + "reserved_bytes_peak": 175389016064, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-01T12:34:28.068343Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110790608384, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11890911232, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 175389016064, + "reserved_bytes_peak": 175389016064, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-01T12:34:27.272461Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11890911232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 175389016064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175389016064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 930, + "segment_count_current_delta": 0, + "segment_count_peak": 930, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T123428068733Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.074995Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T123428068733Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T123428068733Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..797250801ed6260400e614beb464703feb73ceac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T123428068733Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da25c4cdf42c3ae02bf93ab78e7a16d4f1e2c76e5eccf1b6fb75e07a3d95f418 +size 241008 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T123428871671Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T123428871671Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a0609c6f41416f178ed19fce2c299e1b8d918ba9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T123428871671Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162761992192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11890911232, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 175389016064, + "reserved_bytes_peak": 175389016064, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-01T12:34:28.871256Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110790608384, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11890911232, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 175389016064, + "reserved_bytes_peak": 175389016064, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-01T12:34:28.075959Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11890911232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 175389016064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175389016064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 930, + "segment_count_current_delta": 0, + "segment_count_peak": 930, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T123428871671Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.878138Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T123428871671Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T123428871671Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fa06eb45ae6f7f7757f42a41525459a763817775 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T123428871671Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:889ef706f416bd4ea81b5159437edc5a701e8a205d5a515b5851e8039536fcbc +size 241039 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T123529587141Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T123529587141Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b227e68195b836ef811980056a78a0f0baef8af2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T123529587141Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 162761992192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5583301632, + "inactive_split_bytes_peak": 11890911232, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 178396332032, + "reserved_bytes_peak": 178396332032, + "segment_count_current": 944, + "segment_count_peak": 944, + "timestamp_utc": "2026-08-01T12:35:29.586647Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110790608384, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162761992192, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110790608384, + "device": 2, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11890911232, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 175389016064, + "reserved_bytes_peak": 175389016064, + "segment_count_current": 930, + "segment_count_peak": 930, + "timestamp_utc": "2026-08-01T12:34:28.879075Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5583301632, + "inactive_split_bytes_current_delta": -27412480, + "inactive_split_bytes_peak": 11890911232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 178396332032, + "reserved_bytes_current_delta": 3007315968, + "reserved_bytes_peak": 178396332032, + "reserved_bytes_peak_delta": 3007315968, + "segment_count_current": 944, + "segment_count_current_delta": 14, + "segment_count_peak": 944, + "segment_count_peak_delta": 14, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T123529587141Z.pickle", + "timestamp_utc": "2026-08-01T12:35:29.675560Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T123529587141Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T123529587141Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0874d944fcba2c614c3fc5e38c28d651a430e5c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T123529587141Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5397e41a3ccb3780d6b56a98155a97ec05c718e76aab62cfd4da79847b82e53 +size 4080504 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T123419486957Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T123419486957Z.json new file mode 100644 index 0000000000000000000000000000000000000000..99488be3751bf864d0c79c3d4381445c5fe5e27c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T123419486957Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 87155254272, + "active_bytes_peak_delta": 51688918016, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87155254272, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87155254272, + "device": 3, + "inactive_split_bytes_current": 5738906624, + "inactive_split_bytes_peak": 8604603904, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 96871645184, + "reserved_bytes_peak": 96871645184, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T12:34:19.486356Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 87155254272, + "allocated_bytes_peak_delta": 51688918016, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:33:11.535550Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5738906624, + "inactive_split_bytes_current_delta": 5606087680, + "inactive_split_bytes_peak": 8604603904, + "inactive_split_bytes_peak_delta": 8444217856, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 96871645184, + "reserved_bytes_current_delta": 57627639808, + "reserved_bytes_peak": 96871645184, + "reserved_bytes_peak_delta": 57627639808, + "segment_count_current": 502, + "segment_count_current_delta": 431, + "segment_count_peak": 502, + "segment_count_peak_delta": 431, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T123419486957Z.pickle", + "timestamp_utc": "2026-08-01T12:34:19.599689Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T123419486957Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T123419486957Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e2f36301b8ecb16eae764d8506e67a79b02d1dbe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T123419486957Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bce069731f3182933f40ceeca11a84e220bc5b3efab0d1d9c62e9b2d9a444dc4 +size 5398297 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T123420636373Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T123420636373Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e4a9a4cf6fb724bdae28b97c63bde1009ad4998 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T123420636373Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110777787392, + "active_bytes_peak_delta": 23622533120, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110777787392, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110777787392, + "device": 3, + "inactive_split_bytes_current": 5757780992, + "inactive_split_bytes_peak": 8604603904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 114072485888, + "reserved_bytes_peak": 114072485888, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T12:34:20.635957Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110777787392, + "allocated_bytes_peak_delta": 23622533120, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87155254272, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87155254272, + "device": 3, + "inactive_split_bytes_current": 5738906624, + "inactive_split_bytes_peak": 8604603904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 96871645184, + "reserved_bytes_peak": 96871645184, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T12:34:19.619871Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5757780992, + "inactive_split_bytes_current_delta": 18874368, + "inactive_split_bytes_peak": 8604603904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 114072485888, + "reserved_bytes_current_delta": 17200840704, + "reserved_bytes_peak": 114072485888, + "reserved_bytes_peak_delta": 17200840704, + "segment_count_current": 570, + "segment_count_current_delta": 68, + "segment_count_peak": 570, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T123420636373Z.pickle", + "timestamp_utc": "2026-08-01T12:34:20.743811Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T123420636373Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T123420636373Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aa4dadcd42b05ef47da46111a61ef801a273a8e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T123420636373Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c090d759273190198d39dbf7df3257f945843e6c7ef20b6545b82a4e644d7f1 +size 5650448 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T123421749170Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T123421749170Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8de96e0c3d802afb2a513b8463802f77ce85cf9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T123421749170Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110777787392, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110777787392, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110777787392, + "device": 3, + "inactive_split_bytes_current": 5288018944, + "inactive_split_bytes_peak": 8604603904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 115012009984, + "reserved_bytes_peak": 115012009984, + "segment_count_current": 574, + "segment_count_peak": 574, + "timestamp_utc": "2026-08-01T12:34:21.748753Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110777787392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110777787392, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110777787392, + "device": 3, + "inactive_split_bytes_current": 5757780992, + "inactive_split_bytes_peak": 8604603904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 114072485888, + "reserved_bytes_peak": 114072485888, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T12:34:20.759135Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5288018944, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 8604603904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 115012009984, + "reserved_bytes_current_delta": 939524096, + "reserved_bytes_peak": 115012009984, + "reserved_bytes_peak_delta": 939524096, + "segment_count_current": 574, + "segment_count_current_delta": 4, + "segment_count_peak": 574, + "segment_count_peak_delta": 4, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T123421749170Z.pickle", + "timestamp_utc": "2026-08-01T12:34:21.851890Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T123421749170Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T123421749170Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76b472d037220e1721e179f874be50ea2b1e362f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T123421749170Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1aca3c574b40717fb251cfff218b03602b9292cd43eefb437363085bdce5ffa7 +size 5588916 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T123424716262Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T123424716262Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee1c9faac448388b3727f41625a028b50a27d28c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T123424716262Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 51983156224, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_peak": 11666384896, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 175512748032, + "reserved_bytes_peak": 175512748032, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:24.715697Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 9937408, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110777787392, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110777787392, + "device": 3, + "inactive_split_bytes_current": 5288018944, + "inactive_split_bytes_peak": 8604603904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 115012009984, + "reserved_bytes_peak": 115012009984, + "segment_count_current": 574, + "segment_count_peak": 574, + "timestamp_utc": "2026-08-01T12:34:21.866498Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_current_delta": 326889472, + "inactive_split_bytes_peak": 11666384896, + "inactive_split_bytes_peak_delta": 3061780992, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 175512748032, + "reserved_bytes_current_delta": 60500738048, + "reserved_bytes_peak": 175512748032, + "reserved_bytes_peak_delta": 60500738048, + "segment_count_current": 933, + "segment_count_current_delta": 359, + "segment_count_peak": 933, + "segment_count_peak_delta": 359, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T123424716262Z.pickle", + "timestamp_utc": "2026-08-01T12:34:24.835126Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T123424716262Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T123424716262Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eb789a08447c17c7c03c3d15ddc9fee77daa792a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T123424716262Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406c367438724c281ea1daddc639bf9f9c966b36f7212dae35adc29d242915ef +size 5871024 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T123425656493Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T123425656493Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5467d55a03333ff3fce78aeea8e1bcdd25470a45 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T123425656493Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_peak": 11666384896, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 175512748032, + "reserved_bytes_peak": 175512748032, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:25.656014Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_peak": 11666384896, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 175512748032, + "reserved_bytes_peak": 175512748032, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:24.851689Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11666384896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 175512748032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175512748032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 933, + "segment_count_current_delta": 0, + "segment_count_peak": 933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T123425656493Z.pickle", + "timestamp_utc": "2026-08-01T12:34:25.663809Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T123425656493Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T123425656493Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf5c3bcd850b1137e569ed76cec41bfd1fd58729 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T123425656493Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:039885e9e4381de288c61dc259b23ffc2e73ec1b87a8926c2991529c8ad695f7 +size 241471 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T123426463808Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T123426463808Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94c61dd6362ff4b87998a43427681b9845407d73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T123426463808Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_peak": 11666384896, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 175512748032, + "reserved_bytes_peak": 175512748032, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:26.463392Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_peak": 11666384896, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 175512748032, + "reserved_bytes_peak": 175512748032, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:25.664800Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11666384896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 175512748032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175512748032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 933, + "segment_count_current_delta": 0, + "segment_count_peak": 933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T123426463808Z.pickle", + "timestamp_utc": "2026-08-01T12:34:26.470625Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T123426463808Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T123426463808Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e72fb11480e2f795641e1969597c6a18991f2901 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T123426463808Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:310945a707f3578cdf56926952fed7b7b9bfd7e2b3bdce577b4ee62505094291 +size 241471 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T123427265059Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T123427265059Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8077a56ecb86a58492c145be5270e9dbc5ac3ef4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T123427265059Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_peak": 11666384896, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 175512748032, + "reserved_bytes_peak": 175512748032, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:27.264655Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_peak": 11666384896, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 175512748032, + "reserved_bytes_peak": 175512748032, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:26.471604Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11666384896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 175512748032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175512748032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 933, + "segment_count_current_delta": 0, + "segment_count_peak": 933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T123427265059Z.pickle", + "timestamp_utc": "2026-08-01T12:34:27.271671Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T123427265059Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T123427265059Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bdacd9b8d323a1b8113abfd350e723ee00e9d13c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T123427265059Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:144f45f10e40922dc1a57a64736988113018e688b725fb4e3356879f3b45c18c +size 241471 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T123428068905Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T123428068905Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a054bb50ec0af9fd5bcce78ead4e7f89a8e1f0af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T123428068905Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_peak": 11666384896, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 175512748032, + "reserved_bytes_peak": 175512748032, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:28.068448Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_peak": 11666384896, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 175512748032, + "reserved_bytes_peak": 175512748032, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:27.272633Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11666384896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 175512748032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175512748032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 933, + "segment_count_current_delta": 0, + "segment_count_peak": 933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T123428068905Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.076681Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T123428068905Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T123428068905Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ee7fc27150aba69c3f183cdb492347161810ec7b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T123428068905Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:085d0d2eb8050f7ccad250d47f297a5900b017091353c9dfffff83c61eb40bce +size 241471 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T123428871733Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T123428871733Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3e2e7dacd12b3a02b92c88732fbda48cec7167f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T123428871733Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_peak": 11666384896, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 175512748032, + "reserved_bytes_peak": 175512748032, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:28.871314Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_peak": 11666384896, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 175512748032, + "reserved_bytes_peak": 175512748032, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:28.077683Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11666384896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 175512748032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175512748032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 933, + "segment_count_current_delta": 0, + "segment_count_peak": 933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T123428871733Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.878526Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T123428871733Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T123428871733Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..159d1f3f814bb78b5f54bbd8c920db453cbad458 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T123428871733Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:815f2d70231cb6bacc8f7cc0ca18225af69d48c093335220d82cf6527f0cdee7 +size 241471 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T123529587199Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T123529587199Z.json new file mode 100644 index 0000000000000000000000000000000000000000..967cab027c687613438f828d0c122bbd99af73e1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T123529587199Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 162760943616, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5352614912, + "inactive_split_bytes_peak": 11666384896, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 177345658880, + "reserved_bytes_peak": 177345658880, + "segment_count_current": 942, + "segment_count_peak": 942, + "timestamp_utc": "2026-08-01T12:35:29.586691Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110787724800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162760943616, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110787724800, + "device": 3, + "inactive_split_bytes_current": 5614908416, + "inactive_split_bytes_peak": 11666384896, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 175512748032, + "reserved_bytes_peak": 175512748032, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:28.879485Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5352614912, + "inactive_split_bytes_current_delta": -262293504, + "inactive_split_bytes_peak": 11666384896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 177345658880, + "reserved_bytes_current_delta": 1832910848, + "reserved_bytes_peak": 177345658880, + "reserved_bytes_peak_delta": 1832910848, + "segment_count_current": 942, + "segment_count_current_delta": 9, + "segment_count_peak": 942, + "segment_count_peak_delta": 9, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T123529587199Z.pickle", + "timestamp_utc": "2026-08-01T12:35:29.683597Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T123529587199Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T123529587199Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c699e49e62b7425441bc664af97b8f465f5aaf33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T123529587199Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af3a752e174d6fefd688974426efd8368e151d8ee2fcf88e72405c42eba4c018 +size 4083666 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T123419486965Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T123419486965Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4acabf828101209016ae1150d5b381e731579bd5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T123419486965Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922650624, + "active_bytes_current_delta": 31690753024, + "active_bytes_peak": 87158152192, + "active_bytes_peak_delta": 51691373568, + "after": { + "active_bytes_current": 66922650624, + "active_bytes_peak": 87158152192, + "allocated_bytes_current": 66922650624, + "allocated_bytes_peak": 87158152192, + "device": 4, + "inactive_split_bytes_current": 5387150336, + "inactive_split_bytes_peak": 9192797184, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 96821313536, + "reserved_bytes_peak": 96821313536, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T12:34:19.486367Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922650624, + "allocated_bytes_current_delta": 31690753024, + "allocated_bytes_peak": 87158152192, + "allocated_bytes_peak_delta": 51691373568, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:33:11.535774Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5387150336, + "inactive_split_bytes_current_delta": 5254773760, + "inactive_split_bytes_peak": 9192797184, + "inactive_split_bytes_peak_delta": 9039387648, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 96821313536, + "reserved_bytes_current_delta": 57577308160, + "reserved_bytes_peak": 96821313536, + "reserved_bytes_peak_delta": 57577308160, + "segment_count_current": 501, + "segment_count_current_delta": 430, + "segment_count_peak": 501, + "segment_count_peak_delta": 430, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T123419486965Z.pickle", + "timestamp_utc": "2026-08-01T12:34:19.597545Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T123419486965Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T123419486965Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c0370323e167a3cdb6c09f5a899302786b6d3fc5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T123419486965Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5f5b183506a3f40c5bdea83e3ed38313c9acc70bbd9d63ad4d86fab54b6582a +size 5404874 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T123420636348Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T123420636348Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c3d8f7132e6a2eec3a806c3fb79d0419468979f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T123420636348Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922650624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110782599680, + "active_bytes_peak_delta": 23624447488, + "after": { + "active_bytes_current": 66922650624, + "active_bytes_peak": 110782599680, + "allocated_bytes_current": 66922650624, + "allocated_bytes_peak": 110782599680, + "device": 4, + "inactive_split_bytes_current": 5175337984, + "inactive_split_bytes_peak": 9192797184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 114192023552, + "reserved_bytes_peak": 114192023552, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T12:34:20.635941Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922650624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110782599680, + "allocated_bytes_peak_delta": 23624447488, + "before": { + "active_bytes_current": 66922650624, + "active_bytes_peak": 87158152192, + "allocated_bytes_current": 66922650624, + "allocated_bytes_peak": 87158152192, + "device": 4, + "inactive_split_bytes_current": 5387150336, + "inactive_split_bytes_peak": 9192797184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 96821313536, + "reserved_bytes_peak": 96821313536, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T12:34:19.613843Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5175337984, + "inactive_split_bytes_current_delta": -211812352, + "inactive_split_bytes_peak": 9192797184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 114192023552, + "reserved_bytes_current_delta": 17370710016, + "reserved_bytes_peak": 114192023552, + "reserved_bytes_peak_delta": 17370710016, + "segment_count_current": 570, + "segment_count_current_delta": 69, + "segment_count_peak": 570, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T123420636348Z.pickle", + "timestamp_utc": "2026-08-01T12:34:20.740233Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T123420636348Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T123420636348Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f97b4f64a3875f908cb549343e7e5399ca9ae37 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T123420636348Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27d9f4c53bbdeea4c0c9e31d73a4b323f4d747f9d918a27c1f3ed97d0ba857fd +size 5657922 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T123421749165Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T123421749165Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e948b6c19166da1abb4d883199298560fe619349 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T123421749165Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922650624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110785482752, + "active_bytes_peak_delta": 2883072, + "after": { + "active_bytes_current": 66922650624, + "active_bytes_peak": 110785482752, + "allocated_bytes_current": 66922650624, + "allocated_bytes_peak": 110785482752, + "device": 4, + "inactive_split_bytes_current": 4936262656, + "inactive_split_bytes_peak": 9192797184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 114896666624, + "reserved_bytes_peak": 114896666624, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T12:34:21.748763Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922650624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110785482752, + "allocated_bytes_peak_delta": 2883072, + "before": { + "active_bytes_current": 66922650624, + "active_bytes_peak": 110782599680, + "allocated_bytes_current": 66922650624, + "allocated_bytes_peak": 110782599680, + "device": 4, + "inactive_split_bytes_current": 5175337984, + "inactive_split_bytes_peak": 9192797184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 114192023552, + "reserved_bytes_peak": 114192023552, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T12:34:20.752403Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4936262656, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 9192797184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 114896666624, + "reserved_bytes_current_delta": 704643072, + "reserved_bytes_peak": 114896666624, + "reserved_bytes_peak_delta": 704643072, + "segment_count_current": 573, + "segment_count_current_delta": 3, + "segment_count_peak": 573, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T123421749165Z.pickle", + "timestamp_utc": "2026-08-01T12:34:21.845782Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T123421749165Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T123421749165Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..30e7ea027c476a3068795b68c98b19e9b1b544d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T123421749165Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e9da73cc2ae38f4a59ba80291e391dc65c083b2ce58be80a07f8da289faa7e4 +size 5593773 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T123424716198Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T123424716198Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fbf525f26fab8d973f1604432c4ef26d8576c676 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T123424716198Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939694080, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162767194112, + "active_bytes_peak_delta": 51981711360, + "after": { + "active_bytes_current": 66939694080, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66939694080, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_peak": 11436132352, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 175045083136, + "reserved_bytes_peak": 175045083136, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:34:24.715677Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939694080, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110792665088, + "allocated_bytes_peak_delta": 7182336, + "before": { + "active_bytes_current": 66922650624, + "active_bytes_peak": 110785482752, + "allocated_bytes_current": 66922650624, + "allocated_bytes_peak": 110785482752, + "device": 4, + "inactive_split_bytes_current": 4936262656, + "inactive_split_bytes_peak": 9192797184, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 114896666624, + "reserved_bytes_peak": 114896666624, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T12:34:21.857097Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_current_delta": 328986624, + "inactive_split_bytes_peak": 11436132352, + "inactive_split_bytes_peak_delta": 2243335168, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 175045083136, + "reserved_bytes_current_delta": 60148416512, + "reserved_bytes_peak": 175045083136, + "reserved_bytes_peak_delta": 60148416512, + "segment_count_current": 928, + "segment_count_current_delta": 355, + "segment_count_peak": 928, + "segment_count_peak_delta": 355, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T123424716198Z.pickle", + "timestamp_utc": "2026-08-01T12:34:24.834638Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T123424716198Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T123424716198Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e8b15da9c5fa0fc65cdb42418a5d4345aa7d544b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T123424716198Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d708f1e093bfa776edf19843a5dafbec511b78520541b257f255275b76c87e3 +size 6613448 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T123425656468Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T123425656468Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6892bfebb4e234a1cbbf84b0596446c77f9e5afb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T123425656468Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939694080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162767194112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939694080, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66939694080, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_peak": 11436132352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 175045083136, + "reserved_bytes_peak": 175045083136, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:34:25.656012Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939694080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110792665088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939694080, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66939694080, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_peak": 11436132352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 175045083136, + "reserved_bytes_peak": 175045083136, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:34:24.848295Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11436132352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 175045083136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175045083136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T123425656468Z.pickle", + "timestamp_utc": "2026-08-01T12:34:25.663085Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T123425656468Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T123425656468Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2fb7cf83904b3a1f14389a4d7df1e2b764ed9944 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T123425656468Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dde88dd35d9fe4514ad245b218fd032b5566f8f62a469d0bb92f76d55f286c68 +size 272450 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T123426463864Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T123426463864Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fab972c4d393b404ae6d0caaabc4176216840d51 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T123426463864Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939694080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162767194112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939694080, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66939694080, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_peak": 11436132352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 175045083136, + "reserved_bytes_peak": 175045083136, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:34:26.463454Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939694080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110792665088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939694080, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66939694080, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_peak": 11436132352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 175045083136, + "reserved_bytes_peak": 175045083136, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:34:25.664050Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11436132352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 175045083136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175045083136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T123426463864Z.pickle", + "timestamp_utc": "2026-08-01T12:34:26.471095Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T123426463864Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T123426463864Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..412f7efa407a28e0a09f14e658687daa1d707cb4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T123426463864Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ff06ae087b4378093ec10db86030ab43f1b0d196cc7e685b1d0bb9e19d44b11 +size 272450 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T123427265046Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T123427265046Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0116bbb034973b1eb2d3ac6162032b0e20f96ee6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T123427265046Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939694080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162767194112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939694080, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66939694080, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_peak": 11436132352, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 175045083136, + "reserved_bytes_peak": 175045083136, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:34:27.264656Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939694080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110792665088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939694080, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66939694080, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_peak": 11436132352, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 175045083136, + "reserved_bytes_peak": 175045083136, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:34:26.472061Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11436132352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 175045083136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175045083136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T123427265046Z.pickle", + "timestamp_utc": "2026-08-01T12:34:27.277272Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T123427265046Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T123427265046Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ac0d3f3b509b22c05464e2ccd28f446289cced1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T123427265046Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab70a6d8dcc338d0fa05dd026493aafb7816ea62f9842036d4b31998cde5544e +size 272450 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T123428068873Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T123428068873Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e564a0866ad527c6619209def1109ed92613c1b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T123428068873Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939694080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162767194112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939694080, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66939694080, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_peak": 11436132352, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 175045083136, + "reserved_bytes_peak": 175045083136, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:34:28.068447Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939694080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110792665088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939694080, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66939694080, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_peak": 11436132352, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 175045083136, + "reserved_bytes_peak": 175045083136, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:34:27.278300Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11436132352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 175045083136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175045083136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T123428068873Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.076578Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T123428068873Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T123428068873Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe5c65065e01f084e9587165812feb26e571e39d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T123428068873Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:386be99a6ac0d674bcc1181685056b71c5a7f0818df5004d170c8be2e93aa062 +size 272450 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T123428871685Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T123428871685Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6bad7fbeb9c4dfed06b1e5cb31cc22f365968e8b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T123428871685Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939694080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162767194112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939694080, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66939694080, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_peak": 11436132352, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 175045083136, + "reserved_bytes_peak": 175045083136, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:34:28.871295Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939694080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110792665088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939694080, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66939694080, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_peak": 11436132352, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 175045083136, + "reserved_bytes_peak": 175045083136, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:34:28.077543Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11436132352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 175045083136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175045083136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 928, + "segment_count_current_delta": 0, + "segment_count_peak": 928, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T123428871685Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.878475Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T123428871685Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T123428871685Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..63d04347e1fe065ed653646bc8b8581b3de29771 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T123428871685Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5d1a7baa954d9bc1396ecef8f09432bc35c211b9da6a60e0d96ec6433e8acd6 +size 272450 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T123529587146Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T123529587146Z.json new file mode 100644 index 0000000000000000000000000000000000000000..596590b08202d52a394a30f59e4176bf4cfab46f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T123529587146Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66956620800, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 162767194112, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66956620800, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66956620800, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5000858624, + "inactive_split_bytes_peak": 11436132352, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 177347756032, + "reserved_bytes_peak": 177347756032, + "segment_count_current": 939, + "segment_count_peak": 939, + "timestamp_utc": "2026-08-01T12:35:29.586655Z", + "world_size": 8 + }, + "allocated_bytes_current": 66956620800, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110792665088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939694080, + "active_bytes_peak": 162767194112, + "allocated_bytes_current": 66939694080, + "allocated_bytes_peak": 110792665088, + "device": 4, + "inactive_split_bytes_current": 5265249280, + "inactive_split_bytes_peak": 11436132352, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 175045083136, + "reserved_bytes_peak": 175045083136, + "segment_count_current": 928, + "segment_count_peak": 928, + "timestamp_utc": "2026-08-01T12:34:28.879424Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5000858624, + "inactive_split_bytes_current_delta": -264390656, + "inactive_split_bytes_peak": 11436132352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 177347756032, + "reserved_bytes_current_delta": 2302672896, + "reserved_bytes_peak": 177347756032, + "reserved_bytes_peak_delta": 2302672896, + "segment_count_current": 939, + "segment_count_current_delta": 11, + "segment_count_peak": 939, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T123529587146Z.pickle", + "timestamp_utc": "2026-08-01T12:35:29.673077Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T123529587146Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T123529587146Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de416764cbcd5c1bd589ecffbc8650285874815f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T123529587146Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aecff45351aa4d53949b176bf120494359e857275dfb67a955320657e9c7a280 +size 4220003 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T123419487128Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T123419487128Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e9d3c7deee0858bbe6ed919e1beb8910e7d15164 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T123419487128Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 87155254272, + "active_bytes_peak_delta": 51688918016, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87155254272, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87155254272, + "device": 5, + "inactive_split_bytes_current": 5504025600, + "inactive_split_bytes_peak": 8369722880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 96401883136, + "reserved_bytes_peak": 96401883136, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T12:34:19.486519Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 87155254272, + "allocated_bytes_peak_delta": 51688918016, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:33:11.536407Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5504025600, + "inactive_split_bytes_current_delta": 5371206656, + "inactive_split_bytes_peak": 8369722880, + "inactive_split_bytes_peak_delta": 8209336832, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 96401883136, + "reserved_bytes_current_delta": 57157877760, + "reserved_bytes_peak": 96401883136, + "reserved_bytes_peak_delta": 57157877760, + "segment_count_current": 500, + "segment_count_current_delta": 429, + "segment_count_peak": 500, + "segment_count_peak_delta": 429, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T123419487128Z.pickle", + "timestamp_utc": "2026-08-01T12:34:19.600752Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T123419487128Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T123419487128Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..17aae8017dc0e1f6cc79893d38acb7e5eacd3c28 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T123419487128Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9545adb747fbe7f0adc17ef430d6640861af9d279e0392d8cac46744b581777b +size 5397890 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T123420636504Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T123420636504Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7db59756ce5e60cd08573a22f930cb6f4fdb9c74 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T123420636504Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110777787392, + "active_bytes_peak_delta": 23622533120, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110777787392, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110777787392, + "device": 5, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_peak": 8369722880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113602723840, + "reserved_bytes_peak": 113602723840, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T12:34:20.636091Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110777787392, + "allocated_bytes_peak_delta": 23622533120, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87155254272, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87155254272, + "device": 5, + "inactive_split_bytes_current": 5504025600, + "inactive_split_bytes_peak": 8369722880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 96401883136, + "reserved_bytes_peak": 96401883136, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T12:34:19.619085Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_current_delta": -213909504, + "inactive_split_bytes_peak": 8369722880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113602723840, + "reserved_bytes_current_delta": 17200840704, + "reserved_bytes_peak": 113602723840, + "reserved_bytes_peak_delta": 17200840704, + "segment_count_current": 568, + "segment_count_current_delta": 68, + "segment_count_peak": 568, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T123420636504Z.pickle", + "timestamp_utc": "2026-08-01T12:34:20.745074Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T123420636504Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T123420636504Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9cab5e4e59da1289a28deed94736f0e24d3c9dc6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T123420636504Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e10ddd1d1ae13fb283f29193172b4bf023cb7a73e9640eccc7e569580daa1b1 +size 5649906 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T123421749308Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T123421749308Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bea6e038242c18cf3e16350d5bec66d10ed77848 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T123421749308Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110777787392, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110777787392, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110777787392, + "device": 5, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_peak": 8369722880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113602723840, + "reserved_bytes_peak": 113602723840, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T12:34:21.748893Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110777787392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110777787392, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110777787392, + "device": 5, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_peak": 8369722880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113602723840, + "reserved_bytes_peak": 113602723840, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T12:34:20.763058Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8369722880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113602723840, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113602723840, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T123421749308Z.pickle", + "timestamp_utc": "2026-08-01T12:34:21.858256Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T123421749308Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T123421749308Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e9f89ef1bdf26621b0de5fa8e113e16e47da4a0a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T123421749308Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18390f9941256c055dea4699a290f38c2954b23062361d8b50f22cdae31056c9 +size 5590271 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T123424716571Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T123424716571Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bce0330052d07c6a9b59ef9e5a54e5673519cc50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T123424716571Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162769594368, + "active_bytes_peak_delta": 51991806976, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 12200765440, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 175611314176, + "reserved_bytes_peak": 175611314176, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:24.715915Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110795065344, + "allocated_bytes_peak_delta": 17277952, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110777787392, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110777787392, + "device": 5, + "inactive_split_bytes_current": 5290116096, + "inactive_split_bytes_peak": 8369722880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113602723840, + "reserved_bytes_peak": 113602723840, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T12:34:21.875941Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 320598016, + "inactive_split_bytes_peak": 12200765440, + "inactive_split_bytes_peak_delta": 3831042560, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 175611314176, + "reserved_bytes_current_delta": 62008590336, + "reserved_bytes_peak": 175611314176, + "reserved_bytes_peak_delta": 62008590336, + "segment_count_current": 934, + "segment_count_current_delta": 366, + "segment_count_peak": 934, + "segment_count_peak_delta": 366, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T123424716571Z.pickle", + "timestamp_utc": "2026-08-01T12:34:24.836480Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T123424716571Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T123424716571Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..736dd1040a427fb1d4168b8bb64834292080087d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T123424716571Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c815a2b4c5bacd6985c41c5669d3759f2b8e4cf4a88d8e41680195c2f48be76c +size 5870779 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T123425656598Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T123425656598Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0bc76a103a85acb62335b108f26bb2057ceea312 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T123425656598Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162769594368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 12200765440, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 175611314176, + "reserved_bytes_peak": 175611314176, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:25.656124Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110795065344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 12200765440, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 175611314176, + "reserved_bytes_peak": 175611314176, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:24.852868Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12200765440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 175611314176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175611314176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T123425656598Z.pickle", + "timestamp_utc": "2026-08-01T12:34:25.663694Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T123425656598Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T123425656598Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..217707f3cb25b1026d0bc0ce62b1e5760477c1a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T123425656598Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b94e71236f50820a9a66ea929a2f65b2a3c4de22c7164f7e6fde27260da512c1 +size 241875 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T123426463882Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T123426463882Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3b4fa6d3dc9e9a12ffad8c5df88f15f730fcb52e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T123426463882Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162769594368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 12200765440, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 175611314176, + "reserved_bytes_peak": 175611314176, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:26.463471Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110795065344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 12200765440, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 175611314176, + "reserved_bytes_peak": 175611314176, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:25.664664Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12200765440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 175611314176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175611314176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T123426463882Z.pickle", + "timestamp_utc": "2026-08-01T12:34:26.470530Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T123426463882Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T123426463882Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e3b33d68f3ab6f0d57b532add568cf3b61bec3d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T123426463882Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:795017486448bef6833b6e1eeb4c53c0c10567b97fb3eaaf7b65a691c5c678e3 +size 241844 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T123427265139Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T123427265139Z.json new file mode 100644 index 0000000000000000000000000000000000000000..147f8f4850c864abe112ae7832a90b5f21758953 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T123427265139Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162769594368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 12200765440, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 175611314176, + "reserved_bytes_peak": 175611314176, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:27.264736Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110795065344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 12200765440, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 175611314176, + "reserved_bytes_peak": 175611314176, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:26.471483Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12200765440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 175611314176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175611314176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T123427265139Z.pickle", + "timestamp_utc": "2026-08-01T12:34:27.271417Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T123427265139Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T123427265139Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2aaa017253b42240820bb058d0f47dcc3f944712 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T123427265139Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac8b998c209d3b9271e5a95af12f0a3b3b315813c57706cf2a95cfb036d81094 +size 241875 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T123428068881Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T123428068881Z.json new file mode 100644 index 0000000000000000000000000000000000000000..defaa049434ff3389cc82b0e253ffae668479245 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T123428068881Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162769594368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 12200765440, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 175611314176, + "reserved_bytes_peak": 175611314176, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:28.068476Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110795065344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 12200765440, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 175611314176, + "reserved_bytes_peak": 175611314176, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:27.272369Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12200765440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 175611314176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175611314176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T123428068881Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.075184Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T123428068881Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T123428068881Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b5d573c977daa58ba8891a96a11ed55fdb87c1d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T123428068881Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d5e049910ad480395ad56841c6803ed34080a7eebe8aa01e55a2294d9ee9eaf +size 241844 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T123428871767Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T123428871767Z.json new file mode 100644 index 0000000000000000000000000000000000000000..616d4387521ea042ed4379867c07c64d6ec54d33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T123428871767Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162769594368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 12200765440, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 175611314176, + "reserved_bytes_peak": 175611314176, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:28.871364Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110795065344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 12200765440, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 175611314176, + "reserved_bytes_peak": 175611314176, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:28.076117Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 12200765440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 175611314176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175611314176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 934, + "segment_count_current_delta": 0, + "segment_count_peak": 934, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T123428871767Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.878295Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T123428871767Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T123428871767Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..229e5846355d3752b29d235f17e7151f73f5973f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T123428871767Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:958edcdbd0d9e345a24feda72862a3cdc72579f2f668b8e0c8c3a2f085676cd0 +size 241875 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T123529587142Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T123529587142Z.json new file mode 100644 index 0000000000000000000000000000000000000000..747dd1137b846b404e66358bc2c6b675729640f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T123529587142Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 162769594368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5348420608, + "inactive_split_bytes_peak": 12200765440, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 177913987072, + "reserved_bytes_peak": 177913987072, + "segment_count_current": 945, + "segment_count_peak": 945, + "timestamp_utc": "2026-08-01T12:35:29.586648Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110795065344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162769594368, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110795065344, + "device": 5, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 12200765440, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 175611314176, + "reserved_bytes_peak": 175611314176, + "segment_count_current": 934, + "segment_count_peak": 934, + "timestamp_utc": "2026-08-01T12:34:28.879228Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5348420608, + "inactive_split_bytes_current_delta": -262293504, + "inactive_split_bytes_peak": 12200765440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 177913987072, + "reserved_bytes_current_delta": 2302672896, + "reserved_bytes_peak": 177913987072, + "reserved_bytes_peak_delta": 2302672896, + "segment_count_current": 945, + "segment_count_current_delta": 11, + "segment_count_peak": 945, + "segment_count_peak_delta": 11, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T123529587142Z.pickle", + "timestamp_utc": "2026-08-01T12:35:29.674256Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T123529587142Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T123529587142Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..88dd8e18f24198797c8ed2b075c1eed353e7e8d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T123529587142Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b759edd2505f0def9bcccf09d3e0cab1ed208f624ced2ac7875ca8d9ceb35a90 +size 4080678 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T123419486976Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T123419486976Z.json new file mode 100644 index 0000000000000000000000000000000000000000..990b22bc78229bd19cf7ab2cb45383b826469f5b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T123419486976Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 87155254272, + "active_bytes_peak_delta": 51688918016, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87155254272, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87155254272, + "device": 6, + "inactive_split_bytes_current": 5738906624, + "inactive_split_bytes_peak": 8604603904, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 96871645184, + "reserved_bytes_peak": 96871645184, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T12:34:19.486360Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 87155254272, + "allocated_bytes_peak_delta": 51688918016, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:33:11.535787Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5738906624, + "inactive_split_bytes_current_delta": 5606087680, + "inactive_split_bytes_peak": 8604603904, + "inactive_split_bytes_peak_delta": 8444217856, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 96871645184, + "reserved_bytes_current_delta": 57627639808, + "reserved_bytes_peak": 96871645184, + "reserved_bytes_peak_delta": 57627639808, + "segment_count_current": 502, + "segment_count_current_delta": 431, + "segment_count_peak": 502, + "segment_count_peak_delta": 431, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T123419486976Z.pickle", + "timestamp_utc": "2026-08-01T12:34:19.603333Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T123419486976Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T123419486976Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27bffcc1edcb096ad03280ad06c3b07cc7dbf752 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T123419486976Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b7ee19c463d7988eea67d27d1e0a6e132be3df7dd1fa82715a30891285f5fe4 +size 5398296 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T123420636356Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T123420636356Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc0e25563ed52d0869610f80bbe157dad772d8c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T123420636356Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110774482944, + "active_bytes_peak_delta": 23619228672, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110774482944, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110774482944, + "device": 6, + "inactive_split_bytes_current": 5522899968, + "inactive_split_bytes_peak": 8604603904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 114240258048, + "reserved_bytes_peak": 114240258048, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T12:34:20.635931Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110774482944, + "allocated_bytes_peak_delta": 23619228672, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 87155254272, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 87155254272, + "device": 6, + "inactive_split_bytes_current": 5738906624, + "inactive_split_bytes_peak": 8604603904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 96871645184, + "reserved_bytes_peak": 96871645184, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T12:34:19.622958Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5522899968, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 8604603904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 114240258048, + "reserved_bytes_current_delta": 17368612864, + "reserved_bytes_peak": 114240258048, + "reserved_bytes_peak_delta": 17368612864, + "segment_count_current": 570, + "segment_count_current_delta": 68, + "segment_count_peak": 570, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T123420636356Z.pickle", + "timestamp_utc": "2026-08-01T12:34:20.741597Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T123420636356Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T123420636356Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d8e54cb7b5377af2aeb50fb94b6bf37ece0b23ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T123420636356Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a926d48de826c5a15affa9e9dc623999363d06fe40c55081ef8d19060fc24917 +size 5650946 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T123421749151Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T123421749151Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3dd93315aa43b65f51914100bd9e9c99da36c91 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T123421749151Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110774482944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110774482944, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110774482944, + "device": 6, + "inactive_split_bytes_current": 5524997120, + "inactive_split_bytes_peak": 8604603904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 114944901120, + "reserved_bytes_peak": 114944901120, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T12:34:21.748750Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110774482944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110774482944, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110774482944, + "device": 6, + "inactive_split_bytes_current": 5522899968, + "inactive_split_bytes_peak": 8604603904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 114240258048, + "reserved_bytes_peak": 114240258048, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T12:34:20.756520Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5524997120, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 8604603904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 114944901120, + "reserved_bytes_current_delta": 704643072, + "reserved_bytes_peak": 114944901120, + "reserved_bytes_peak_delta": 704643072, + "segment_count_current": 573, + "segment_count_current_delta": 3, + "segment_count_peak": 573, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T123421749151Z.pickle", + "timestamp_utc": "2026-08-01T12:34:21.853357Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T123421749151Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T123421749151Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..71ea93bd810de2841bf3728418fe1fbd0f4b8f7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T123421749151Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d641412bf8fdf21f8f513f26ecb3ee7dbcb84fee2674eaf5c16f5524c729a028 +size 5589609 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T123424716279Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T123424716279Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e5f01f914d2caa7ade389c4bf9f7d511e197eba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T123424716279Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162768021504, + "active_bytes_peak_delta": 51993538560, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11920082944, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 175720366080, + "reserved_bytes_peak": 175720366080, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:24.715757Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110793492480, + "allocated_bytes_peak_delta": 19009536, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 110774482944, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 110774482944, + "device": 6, + "inactive_split_bytes_current": 5524997120, + "inactive_split_bytes_peak": 8604603904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 114944901120, + "reserved_bytes_peak": 114944901120, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T12:34:21.868028Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 85716992, + "inactive_split_bytes_peak": 11920082944, + "inactive_split_bytes_peak_delta": 3315479040, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 175720366080, + "reserved_bytes_current_delta": 60775464960, + "reserved_bytes_peak": 175720366080, + "reserved_bytes_peak_delta": 60775464960, + "segment_count_current": 933, + "segment_count_current_delta": 360, + "segment_count_peak": 933, + "segment_count_peak_delta": 360, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T123424716279Z.pickle", + "timestamp_utc": "2026-08-01T12:34:24.842073Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T123424716279Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T123424716279Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3cdb0f5fa3dddf12e769b4da31887ea7703413af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T123424716279Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7970e4b69c81746daf07a0e159501d3eda088063f521dbe06169a1b6bb60671 +size 5870466 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T123425656497Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T123425656497Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d26b59fb5451498d8bf98249bc10fba1b4c5d0a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T123425656497Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162768021504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11920082944, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 175720366080, + "reserved_bytes_peak": 175720366080, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:25.656011Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110793492480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11920082944, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 175720366080, + "reserved_bytes_peak": 175720366080, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:24.861505Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11920082944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 175720366080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175720366080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 933, + "segment_count_current_delta": 0, + "segment_count_peak": 933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T123425656497Z.pickle", + "timestamp_utc": "2026-08-01T12:34:25.663577Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T123425656497Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T123425656497Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2eecc0755e1ece7b0f6778832b8a7c78603fe16b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T123425656497Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b42967ca8bcefea524c33308fd8f0981325d35ad4c5fd340937b96551f352c8 +size 240604 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T123426463766Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T123426463766Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e17f63f615edece4191c0c1e646983c619fead02 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T123426463766Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162768021504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11920082944, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 175720366080, + "reserved_bytes_peak": 175720366080, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:26.463365Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110793492480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11920082944, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 175720366080, + "reserved_bytes_peak": 175720366080, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:25.664590Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11920082944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 175720366080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175720366080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 933, + "segment_count_current_delta": 0, + "segment_count_peak": 933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T123426463766Z.pickle", + "timestamp_utc": "2026-08-01T12:34:26.470349Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T123426463766Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T123426463766Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..be2a15f8e1f27d1709ebc9e36631d8c36f67181e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T123426463766Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51f99fc73f9d900a3bf3596a3b1f2d316083b970991bbd7113ef962d955e5c2a +size 240573 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T123427265024Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T123427265024Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0517bed5bba3861c76a957b6eb3d9178a5636108 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T123427265024Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162768021504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11920082944, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 175720366080, + "reserved_bytes_peak": 175720366080, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:27.264635Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110793492480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11920082944, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 175720366080, + "reserved_bytes_peak": 175720366080, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:26.471287Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11920082944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 175720366080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175720366080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 933, + "segment_count_current_delta": 0, + "segment_count_peak": 933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T123427265024Z.pickle", + "timestamp_utc": "2026-08-01T12:34:27.271286Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T123427265024Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T123427265024Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..100d21b080b03dcb07c27fab732d5a20d9a393a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T123427265024Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7128450b396154d5939a17e58ed9d7a577026021d196cd8515e6998e47b8c31c +size 240604 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T123428068763Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T123428068763Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4aec68541220bd3aa9fcb3a61d2680e32cab5051 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T123428068763Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162768021504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11920082944, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 175720366080, + "reserved_bytes_peak": 175720366080, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:28.068375Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110793492480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11920082944, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 175720366080, + "reserved_bytes_peak": 175720366080, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:27.272226Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11920082944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 175720366080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175720366080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 933, + "segment_count_current_delta": 0, + "segment_count_peak": 933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T123428068763Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.075020Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T123428068763Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T123428068763Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..96b4f2d4718cc4230f7aac6cfb250c1c99e6909e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T123428068763Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46419724d8c1cac13c5cce0d2c6080974457e21927fb5af9f965914fbcd20ae5 +size 240573 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T123428871815Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T123428871815Z.json new file mode 100644 index 0000000000000000000000000000000000000000..11dfd8da3c6629597f3dd12629d3034e80d3b778 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T123428871815Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162768021504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11920082944, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 175720366080, + "reserved_bytes_peak": 175720366080, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:28.871344Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110793492480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11920082944, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 175720366080, + "reserved_bytes_peak": 175720366080, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:28.075970Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11920082944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 175720366080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175720366080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 933, + "segment_count_current_delta": 0, + "segment_count_peak": 933, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T123428871815Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.878680Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T123428871815Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T123428871815Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2e0495f17e3d7277f05fbba68c049751cb0e629 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T123428871815Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdb71a989cd9dcf295b7b45bd6d333eb0296014c0840f34b8fb7300d7707c7b5 +size 240604 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T123529587123Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T123529587123Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ffc7154ccf719e1f76ba65e4a8f0873250ab781 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T123529587123Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 162768021504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5583301632, + "inactive_split_bytes_peak": 11920082944, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 178492801024, + "reserved_bytes_peak": 178492801024, + "segment_count_current": 946, + "segment_count_peak": 946, + "timestamp_utc": "2026-08-01T12:35:29.586647Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110793492480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 162768021504, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 110793492480, + "device": 6, + "inactive_split_bytes_current": 5610714112, + "inactive_split_bytes_peak": 11920082944, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 175720366080, + "reserved_bytes_peak": 175720366080, + "segment_count_current": 933, + "segment_count_peak": 933, + "timestamp_utc": "2026-08-01T12:34:28.879668Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5583301632, + "inactive_split_bytes_current_delta": -27412480, + "inactive_split_bytes_peak": 11920082944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 178492801024, + "reserved_bytes_current_delta": 2772434944, + "reserved_bytes_peak": 178492801024, + "reserved_bytes_peak_delta": 2772434944, + "segment_count_current": 946, + "segment_count_current_delta": 13, + "segment_count_peak": 946, + "segment_count_peak_delta": 13, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T123529587123Z.pickle", + "timestamp_utc": "2026-08-01T12:35:29.672946Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T123529587123Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T123529587123Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8520bb20abeb28ab2f89c1771066d8ec2a93864c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T123529587123Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e93fd9546345037e75fb3745fe9372accda59b7f3b65ed5b74590462deb5539 +size 4079926 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T123419486950Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T123419486950Z.json new file mode 100644 index 0000000000000000000000000000000000000000..573a0a82a792faf13ebf60ad2363e7329c869959 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T123419486950Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 31689664000, + "active_bytes_peak": 87155254272, + "active_bytes_peak_delta": 51688918016, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 87155254272, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 87155254272, + "device": 7, + "inactive_split_bytes_current": 5736808960, + "inactive_split_bytes_peak": 8371820032, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 96871645184, + "reserved_bytes_peak": 96871645184, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T12:34:19.486441Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 31689664000, + "allocated_bytes_peak": 87155254272, + "allocated_bytes_peak_delta": 51688918016, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:33:11.540751Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5736808960, + "inactive_split_bytes_current_delta": 5603990016, + "inactive_split_bytes_peak": 8371820032, + "inactive_split_bytes_peak_delta": 8211433984, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 96871645184, + "reserved_bytes_current_delta": 57627639808, + "reserved_bytes_peak": 96871645184, + "reserved_bytes_peak_delta": 57627639808, + "segment_count_current": 502, + "segment_count_current_delta": 431, + "segment_count_peak": 502, + "segment_count_peak_delta": 431, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T123419486950Z.pickle", + "timestamp_utc": "2026-08-01T12:34:19.604346Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T123419486950Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T123419486950Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f91bb927f0616d411592b38a003fbb00a658e5d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T123419486950Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3facafe56f124bb37e3c314381ec3e1e65d826644dae0990520417a3a06bf91 +size 5397979 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T123420636302Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T123420636302Z.json new file mode 100644 index 0000000000000000000000000000000000000000..719978506496f461bf263bf4169ceb08333267db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T123420636302Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110774483456, + "active_bytes_peak_delta": 23619229184, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 110774483456, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 110774483456, + "device": 7, + "inactive_split_bytes_current": 5522899456, + "inactive_split_bytes_peak": 8371820032, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 114240258048, + "reserved_bytes_peak": 114240258048, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T12:34:20.635897Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110774483456, + "allocated_bytes_peak_delta": 23619229184, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 87155254272, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 87155254272, + "device": 7, + "inactive_split_bytes_current": 5736808960, + "inactive_split_bytes_peak": 8371820032, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 96871645184, + "reserved_bytes_peak": 96871645184, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T12:34:19.624233Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5522899456, + "inactive_split_bytes_current_delta": -213909504, + "inactive_split_bytes_peak": 8371820032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 114240258048, + "reserved_bytes_current_delta": 17368612864, + "reserved_bytes_peak": 114240258048, + "reserved_bytes_peak_delta": 17368612864, + "segment_count_current": 570, + "segment_count_current_delta": 68, + "segment_count_peak": 570, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T123420636302Z.pickle", + "timestamp_utc": "2026-08-01T12:34:20.743718Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T123420636302Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T123420636302Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ed7cd961452256d2aeddd4fcd2e34b9a5b635707 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T123420636302Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:089ce463704c03fd3d2041b3e7ebd2b3da11102ddd0850576b2f628b6c0d98fd +size 5652796 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T123421749192Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T123421749192Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c17b763ddcfa6a797afb1f82ebccbf3562fb1d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T123421749192Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110774483456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 110774483456, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 110774483456, + "device": 7, + "inactive_split_bytes_current": 5288018432, + "inactive_split_bytes_peak": 8371820032, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 115769081856, + "reserved_bytes_peak": 115769081856, + "segment_count_current": 574, + "segment_count_peak": 574, + "timestamp_utc": "2026-08-01T12:34:21.748781Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110774483456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 110774483456, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 110774483456, + "device": 7, + "inactive_split_bytes_current": 5522899456, + "inactive_split_bytes_peak": 8371820032, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 114240258048, + "reserved_bytes_peak": 114240258048, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T12:34:20.758980Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5288018432, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 8371820032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 115769081856, + "reserved_bytes_current_delta": 1528823808, + "reserved_bytes_peak": 115769081856, + "reserved_bytes_peak_delta": 1528823808, + "segment_count_current": 574, + "segment_count_current_delta": 4, + "segment_count_peak": 574, + "segment_count_peak_delta": 4, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T123421749192Z.pickle", + "timestamp_utc": "2026-08-01T12:34:21.856657Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T123421749192Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T123421749192Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ec2919796eb54e8541e594802bc9c22e3a8437c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T123421749192Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a645abb89270b810898a738c843b531581c6d9c87bcb288180e55ecad5e4dbb +size 5591471 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T123424716436Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T123424716436Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7029a1706fb4af01e295fc5641d3ae3e2594024 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T123424716436Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 162765662720, + "active_bytes_peak_delta": 51991179264, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_peak": 11785790976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 175395307520, + "reserved_bytes_peak": 175395307520, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-01T12:34:24.715822Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 110791133696, + "allocated_bytes_peak_delta": 16650240, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 110774483456, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 110774483456, + "device": 7, + "inactive_split_bytes_current": 5288018432, + "inactive_split_bytes_peak": 8371820032, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 115769081856, + "reserved_bytes_peak": 115769081856, + "segment_count_current": 574, + "segment_count_peak": 574, + "timestamp_utc": "2026-08-01T12:34:21.871419Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_current_delta": 322695168, + "inactive_split_bytes_peak": 11785790976, + "inactive_split_bytes_peak_delta": 3413970944, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 175395307520, + "reserved_bytes_current_delta": 59626225664, + "reserved_bytes_peak": 175395307520, + "reserved_bytes_peak_delta": 59626225664, + "segment_count_current": 932, + "segment_count_current_delta": 358, + "segment_count_peak": 932, + "segment_count_peak_delta": 358, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T123424716436Z.pickle", + "timestamp_utc": "2026-08-01T12:34:24.836912Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T123424716436Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T123424716436Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ab31a0c15183f4f8dbd29b251fb8629cdf70ea1e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T123424716436Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:787f2b68832be17b86eb4ef10c2c4ad8017aebd1c4845d1f59314791ecaafe2e +size 5872529 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T123425656525Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T123425656525Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3733b3cc456d884b70850c87651703a36528c11a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T123425656525Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162765662720, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_peak": 11785790976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 175395307520, + "reserved_bytes_peak": 175395307520, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-01T12:34:25.656039Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791133696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_peak": 11785790976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 175395307520, + "reserved_bytes_peak": 175395307520, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-01T12:34:24.853654Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11785790976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 175395307520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175395307520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 932, + "segment_count_current_delta": 0, + "segment_count_peak": 932, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T123425656525Z.pickle", + "timestamp_utc": "2026-08-01T12:34:25.663730Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T123425656525Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T123425656525Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..012cbf8342d64889be254f9993d69e9c2a4fd4df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T123425656525Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef519a27d6d4c3dd807b2c5e1571841abb50f73092f07a3cdf7444b3f9b939ed +size 243604 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T123426463828Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T123426463828Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d53ec42fa43c575f4f901ec4a8116622f945102e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T123426463828Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162765662720, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_peak": 11785790976, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 175395307520, + "reserved_bytes_peak": 175395307520, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-01T12:34:26.463416Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791133696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_peak": 11785790976, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 175395307520, + "reserved_bytes_peak": 175395307520, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-01T12:34:25.664764Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11785790976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 175395307520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175395307520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 932, + "segment_count_current_delta": 0, + "segment_count_peak": 932, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T123426463828Z.pickle", + "timestamp_utc": "2026-08-01T12:34:26.470641Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T123426463828Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T123426463828Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..70c536c302671d97da5466bdfba55a53ce0f8702 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T123426463828Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5968ff6ea646b73a2d7ff38a21fd93ffc6831fe45e8e2bf35f74e4274da2f588 +size 243604 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T123427265087Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T123427265087Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f82ac218a88de030d181e6adedf3972e8a4edc2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T123427265087Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162765662720, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_peak": 11785790976, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 175395307520, + "reserved_bytes_peak": 175395307520, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-01T12:34:27.264679Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791133696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_peak": 11785790976, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 175395307520, + "reserved_bytes_peak": 175395307520, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-01T12:34:26.471622Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11785790976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 175395307520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175395307520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 932, + "segment_count_current_delta": 0, + "segment_count_peak": 932, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T123427265087Z.pickle", + "timestamp_utc": "2026-08-01T12:34:27.271626Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T123427265087Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T123427265087Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fd591744ef63ff4caf1ceeadbf5ca031a3869c6c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T123427265087Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78bb2cdb50d27eac78643bcfff6c3362c7f3a56766ffa825d07d3e132d3909b8 +size 243573 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T123428068857Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T123428068857Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ada36e321ae0f801b90338dbc90651f567df7589 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T123428068857Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162765662720, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_peak": 11785790976, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 175395307520, + "reserved_bytes_peak": 175395307520, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-01T12:34:28.068424Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791133696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_peak": 11785790976, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 175395307520, + "reserved_bytes_peak": 175395307520, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-01T12:34:27.272572Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11785790976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 175395307520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175395307520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 932, + "segment_count_current_delta": 0, + "segment_count_peak": 932, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T123428068857Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.075890Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T123428068857Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T123428068857Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b54633b339c104b95e6afe5f3ed0dbf82ef317d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T123428068857Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f60d8c9258751ee83ab17428242f7e6f3bba96a043219754e444075a83e3f5a +size 243604 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T123428871750Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T123428871750Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9bf5595f4b395311c746887855941e65a1e7cbbd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T123428871750Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 162765662720, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_peak": 11785790976, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 175395307520, + "reserved_bytes_peak": 175395307520, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-01T12:34:28.871331Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110791133696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_peak": 11785790976, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 175395307520, + "reserved_bytes_peak": 175395307520, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-01T12:34:28.076854Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11785790976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 175395307520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 175395307520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 932, + "segment_count_current_delta": 0, + "segment_count_peak": 932, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T123428871750Z.pickle", + "timestamp_utc": "2026-08-01T12:34:28.878599Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T123428871750Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T123428871750Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1e6a99151b24ac854cd2123ceda81d93678084a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T123428871750Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:234f21701c9070f3ab05ec29a0028743b6cd3012f570488230f940732e0a3f3a +size 243604 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T123529587110Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T123529587110Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ff390ab20d6f94e7029055ba9b5b1d0fe076f996 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T123529587110Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955089408, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 162765662720, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955089408, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66955089408, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5348420096, + "inactive_split_bytes_peak": 11785790976, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 178402623488, + "reserved_bytes_peak": 178402623488, + "segment_count_current": 946, + "segment_count_peak": 946, + "timestamp_utc": "2026-08-01T12:35:29.586651Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955089408, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 110791133696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 162765662720, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 110791133696, + "device": 7, + "inactive_split_bytes_current": 5610713600, + "inactive_split_bytes_peak": 11785790976, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 175395307520, + "reserved_bytes_peak": 175395307520, + "segment_count_current": 932, + "segment_count_peak": 932, + "timestamp_utc": "2026-08-01T12:34:28.879550Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5348420096, + "inactive_split_bytes_current_delta": -262293504, + "inactive_split_bytes_peak": 11785790976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 178402623488, + "reserved_bytes_current_delta": 3007315968, + "reserved_bytes_peak": 178402623488, + "reserved_bytes_peak_delta": 3007315968, + "segment_count_current": 946, + "segment_count_current_delta": 14, + "segment_count_peak": 946, + "segment_count_peak_delta": 14, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T123529587110Z.pickle", + "timestamp_utc": "2026-08-01T12:35:29.676371Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T123529587110Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T123529587110Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3a1042d00d4629a977f8eafef7d37181b17a61a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T123529587110Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75b60d7e8b23b4649ee05bb703beb04981f1fd81d46ecf2bc5b461f156e5f840 +size 4086656 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T123827059230Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T123827059230Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18695a49de5097e9635e2666c15b54d30f585789 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T123827059230Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 87406443520, + "active_bytes_peak_delta": 51940107264, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87406443520, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87054121984, + "device": 0, + "inactive_split_bytes_current": 5368022016, + "inactive_split_bytes_peak": 9984791552, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 97028931584, + "reserved_bytes_peak": 97028931584, + "segment_count_current": 455, + "segment_count_peak": 455, + "timestamp_utc": "2026-08-01T12:38:27.058660Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 87054121984, + "allocated_bytes_peak_delta": 51587785728, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:37:17.649140Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5368022016, + "inactive_split_bytes_current_delta": 5233105920, + "inactive_split_bytes_peak": 9984791552, + "inactive_split_bytes_peak_delta": 9824159744, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 97028931584, + "reserved_bytes_current_delta": 57782829056, + "reserved_bytes_peak": 97028931584, + "reserved_bytes_peak_delta": 57782829056, + "segment_count_current": 455, + "segment_count_current_delta": 383, + "segment_count_peak": 455, + "segment_count_peak_delta": 383, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T123827059230Z.pickle", + "timestamp_utc": "2026-08-01T12:38:27.175907Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T123827059230Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T123827059230Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..24cc8f0edaa1903c5c70d581cb2493b2aae39ca8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T123827059230Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36f665a083175a30cf9289e85736c625b361ee9825ccbae4caf0b28691c83544 +size 5963993 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T123828360540Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T123828360540Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd9895e743005161119eea26c6a072dc80f8aae3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T123828360540Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111381040128, + "active_bytes_peak_delta": 23974596608, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111381040128, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110676397056, + "device": 0, + "inactive_split_bytes_current": 8958346240, + "inactive_split_bytes_peak": 11321304064, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116823949312, + "reserved_bytes_peak": 116823949312, + "segment_count_current": 542, + "segment_count_peak": 542, + "timestamp_utc": "2026-08-01T12:38:28.360127Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110676397056, + "allocated_bytes_peak_delta": 23622275072, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87406443520, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87054121984, + "device": 0, + "inactive_split_bytes_current": 5368022016, + "inactive_split_bytes_peak": 9984791552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 97028931584, + "reserved_bytes_peak": 97028931584, + "segment_count_current": 455, + "segment_count_peak": 455, + "timestamp_utc": "2026-08-01T12:38:27.194463Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8958346240, + "inactive_split_bytes_current_delta": 3590324224, + "inactive_split_bytes_peak": 11321304064, + "inactive_split_bytes_peak_delta": 1336512512, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116823949312, + "reserved_bytes_current_delta": 19795017728, + "reserved_bytes_peak": 116823949312, + "reserved_bytes_peak_delta": 19795017728, + "segment_count_current": 542, + "segment_count_current_delta": 87, + "segment_count_peak": 542, + "segment_count_peak_delta": 87, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T123828360540Z.pickle", + "timestamp_utc": "2026-08-01T12:38:28.473478Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T123828360540Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T123828360540Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9219fbffb14c0b05a0d691cacc95e8a786c7a09d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T123828360540Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef8e0fce47bdbcee9b546d6555d520e40921ee36cba334e48f7f4d0f1aa464d8 +size 6618304 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T123829611201Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T123829611201Z.json new file mode 100644 index 0000000000000000000000000000000000000000..adbd7fc802c5033055f9ea2e809598308df3e506 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T123829611201Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111385057792, + "active_bytes_peak_delta": 4017664, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111385057792, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110680414720, + "device": 0, + "inactive_split_bytes_current": 8371143680, + "inactive_split_bytes_peak": 11321304064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116826046464, + "reserved_bytes_peak": 116826046464, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:38:29.610788Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110680414720, + "allocated_bytes_peak_delta": 4017664, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111381040128, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110676397056, + "device": 0, + "inactive_split_bytes_current": 8958346240, + "inactive_split_bytes_peak": 11321304064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116823949312, + "reserved_bytes_peak": 116823949312, + "segment_count_current": 542, + "segment_count_peak": 542, + "timestamp_utc": "2026-08-01T12:38:28.489520Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8371143680, + "inactive_split_bytes_current_delta": -587202560, + "inactive_split_bytes_peak": 11321304064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116826046464, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 116826046464, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 543, + "segment_count_current_delta": 1, + "segment_count_peak": 543, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T123829611201Z.pickle", + "timestamp_utc": "2026-08-01T12:38:29.721586Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T123829611201Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T123829611201Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7c2dcfb618b613f0a59317fcbfc81ae415851863 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T123829611201Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37fb09b54e02204d86d0dffbc43cbc477a3f917c07d6745d0df0ea01b522759d +size 6625758 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T123831526402Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T123831526402Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a54807cf47257e99787e8ed987d7ce1460decf6d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T123831526402Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 167065139200, + "active_bytes_current_delta": 94557518848, + "active_bytes_peak": 167495924736, + "active_bytes_peak_delta": 56110866944, + "after": { + "active_bytes_current": 167065139200, + "active_bytes_peak": 167495924736, + "allocated_bytes_current": 109453465600, + "allocated_bytes_peak": 110683563520, + "device": 0, + "inactive_split_bytes_current": 10674784256, + "inactive_split_bytes_peak": 18364381696, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 182655647744, + "reserved_bytes_peak": 182655647744, + "segment_count_current": 855, + "segment_count_peak": 855, + "timestamp_utc": "2026-08-01T12:38:31.525730Z", + "world_size": 8 + }, + "allocated_bytes_current": 109453465600, + "allocated_bytes_current_delta": 36945845248, + "allocated_bytes_peak": 110683563520, + "allocated_bytes_peak_delta": 3148800, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111385057792, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110680414720, + "device": 0, + "inactive_split_bytes_current": 8371143680, + "inactive_split_bytes_peak": 11321304064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116826046464, + "reserved_bytes_peak": 116826046464, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T12:38:29.737077Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10674784256, + "inactive_split_bytes_current_delta": 2303640576, + "inactive_split_bytes_peak": 18364381696, + "inactive_split_bytes_peak_delta": 7043077632, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 182655647744, + "reserved_bytes_current_delta": 65829601280, + "reserved_bytes_peak": 182655647744, + "reserved_bytes_peak_delta": 65829601280, + "segment_count_current": 855, + "segment_count_current_delta": 312, + "segment_count_peak": 855, + "segment_count_peak_delta": 312, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T123831526402Z.pickle", + "timestamp_utc": "2026-08-01T12:38:31.664305Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T123831526402Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T123831526402Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d5f26979690f8ffff6f1cd7d4840f3ad86ca8d05 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T123831526402Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99314c1a7b63ae15b575fccf5e4e6a2d247fa2853707fccd76167441427a87d7 +size 6393134 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T123827059274Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T123827059274Z.json new file mode 100644 index 0000000000000000000000000000000000000000..55bcfd38ef7fbe35afbec724a1a757d17b0ef3e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T123827059274Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 87409182720, + "active_bytes_peak_delta": 51942846464, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87409182720, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87056861184, + "device": 1, + "inactive_split_bytes_current": 5307204608, + "inactive_split_bytes_peak": 8734495744, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 97263812608, + "reserved_bytes_peak": 97263812608, + "segment_count_current": 456, + "segment_count_peak": 456, + "timestamp_utc": "2026-08-01T12:38:27.058676Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 87056861184, + "allocated_bytes_peak_delta": 51590524928, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:37:17.649242Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5307204608, + "inactive_split_bytes_current_delta": 5174385664, + "inactive_split_bytes_peak": 8734495744, + "inactive_split_bytes_peak_delta": 8574109696, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 97263812608, + "reserved_bytes_current_delta": 58019807232, + "reserved_bytes_peak": 97263812608, + "reserved_bytes_peak_delta": 58019807232, + "segment_count_current": 456, + "segment_count_current_delta": 385, + "segment_count_peak": 456, + "segment_count_peak_delta": 385, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T123827059274Z.pickle", + "timestamp_utc": "2026-08-01T12:38:27.177018Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T123827059274Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T123827059274Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..10373ddc43fd0215b23982bb83c833cc4a13b101 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T123827059274Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:935b988c00f90f08fc8cb7213c508d91782c56ece830ecc8d2177f55578856bb +size 5955367 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T123828360590Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T123828360590Z.json new file mode 100644 index 0000000000000000000000000000000000000000..527ed5cfc65e9b6613f4a73ae05a2a236d6fedbd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T123828360590Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111726983680, + "active_bytes_peak_delta": 24317800960, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111726983680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110675197952, + "device": 1, + "inactive_split_bytes_current": 8654259200, + "inactive_split_bytes_peak": 10899353600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 115878133760, + "reserved_bytes_peak": 115878133760, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T12:38:28.360177Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110675197952, + "allocated_bytes_peak_delta": 23618336768, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87409182720, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87056861184, + "device": 1, + "inactive_split_bytes_current": 5307204608, + "inactive_split_bytes_peak": 8734495744, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 97263812608, + "reserved_bytes_peak": 97263812608, + "segment_count_current": 456, + "segment_count_peak": 456, + "timestamp_utc": "2026-08-01T12:38:27.192743Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8654259200, + "inactive_split_bytes_current_delta": 3347054592, + "inactive_split_bytes_peak": 10899353600, + "inactive_split_bytes_peak_delta": 2164857856, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 115878133760, + "reserved_bytes_current_delta": 18614321152, + "reserved_bytes_peak": 115878133760, + "reserved_bytes_peak_delta": 18614321152, + "segment_count_current": 533, + "segment_count_current_delta": 77, + "segment_count_peak": 533, + "segment_count_peak_delta": 77, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T123828360590Z.pickle", + "timestamp_utc": "2026-08-01T12:38:28.473463Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T123828360590Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T123828360590Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3853a9525742ab9ba6a9be6597cf88bd111a93de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T123828360590Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11f8d24fb6b0ed627555f78cdd1259cc89ba8a58867d078f67b6bb16e150dff7 +size 6604431 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T123829611213Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T123829611213Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7da2c60d40e029bffd3b33fd58dfd944889fa1b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T123829611213Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111726983680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111726983680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110675197952, + "device": 1, + "inactive_split_bytes_current": 9365193728, + "inactive_split_bytes_peak": 12190390272, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117172076544, + "reserved_bytes_peak": 117172076544, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:38:29.610813Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110675197952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111726983680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110675197952, + "device": 1, + "inactive_split_bytes_current": 8654259200, + "inactive_split_bytes_peak": 10899353600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 115878133760, + "reserved_bytes_peak": 115878133760, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T12:38:28.486282Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9365193728, + "inactive_split_bytes_current_delta": 710934528, + "inactive_split_bytes_peak": 12190390272, + "inactive_split_bytes_peak_delta": 1291036672, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117172076544, + "reserved_bytes_current_delta": 1293942784, + "reserved_bytes_peak": 117172076544, + "reserved_bytes_peak_delta": 1293942784, + "segment_count_current": 535, + "segment_count_current_delta": 2, + "segment_count_peak": 535, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T123829611213Z.pickle", + "timestamp_utc": "2026-08-01T12:38:29.720120Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T123829611213Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T123829611213Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3cdc09195baf9a903c1a0497979ca4629c7de9e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T123829611213Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1aa5c8e57faa8cb81316df591733d76b47aa3a6d8b728d6d4451cdc54a0d2e28 +size 6081414 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T123831295489Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T123831295489Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aaf74e463c240765448d683fe27e495bf26d8552 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T123831295489Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 166266731520, + "active_bytes_current_delta": 93759111168, + "active_bytes_peak": 166266731520, + "active_bytes_peak_delta": 54539747840, + "after": { + "active_bytes_current": 166266731520, + "active_bytes_peak": 166266731520, + "allocated_bytes_current": 109007379456, + "allocated_bytes_peak": 110685398528, + "device": 1, + "inactive_split_bytes_current": 10881795072, + "inactive_split_bytes_peak": 18182191616, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 182701785088, + "reserved_bytes_peak": 182701785088, + "segment_count_current": 846, + "segment_count_peak": 846, + "timestamp_utc": "2026-08-01T12:38:31.294897Z", + "world_size": 8 + }, + "allocated_bytes_current": 109007379456, + "allocated_bytes_current_delta": 36499759104, + "allocated_bytes_peak": 110685398528, + "allocated_bytes_peak_delta": 10200576, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111726983680, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110675197952, + "device": 1, + "inactive_split_bytes_current": 9365193728, + "inactive_split_bytes_peak": 12190390272, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117172076544, + "reserved_bytes_peak": 117172076544, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:38:29.732491Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10881795072, + "inactive_split_bytes_current_delta": 1516601344, + "inactive_split_bytes_peak": 18182191616, + "inactive_split_bytes_peak_delta": 5991801344, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 182701785088, + "reserved_bytes_current_delta": 65529708544, + "reserved_bytes_peak": 182701785088, + "reserved_bytes_peak_delta": 65529708544, + "segment_count_current": 846, + "segment_count_current_delta": 311, + "segment_count_peak": 846, + "segment_count_peak_delta": 311, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T123831295489Z.pickle", + "timestamp_utc": "2026-08-01T12:38:31.414810Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T123831295489Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T123831295489Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..222c52c207d3e4f9cf381fe3c73ef49c45d2b33b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T123831295489Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab76442f749ae4d30b9e8983d43666aeb42e9bbbed8df73278a07ab9ae663697 +size 5606108 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T123827059217Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T123827059217Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f9d4a30e407fffc8196e98041adbac188d92160 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T123827059217Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 87407346688, + "active_bytes_peak_delta": 51941010432, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87407346688, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87055025152, + "device": 2, + "inactive_split_bytes_current": 5424645120, + "inactive_split_bytes_peak": 8908559360, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 97498693632, + "reserved_bytes_peak": 97498693632, + "segment_count_current": 457, + "segment_count_peak": 457, + "timestamp_utc": "2026-08-01T12:38:27.058621Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 87055025152, + "allocated_bytes_peak_delta": 51588688896, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:37:17.649021Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5424645120, + "inactive_split_bytes_current_delta": 5291826176, + "inactive_split_bytes_peak": 8908559360, + "inactive_split_bytes_peak_delta": 8748173312, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 97498693632, + "reserved_bytes_current_delta": 58254688256, + "reserved_bytes_peak": 97498693632, + "reserved_bytes_peak_delta": 58254688256, + "segment_count_current": 457, + "segment_count_current_delta": 386, + "segment_count_peak": 457, + "segment_count_peak_delta": 386, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T123827059217Z.pickle", + "timestamp_utc": "2026-08-01T12:38:27.179543Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T123827059217Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T123827059217Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aaab15696613bb05868e8a4c564aa028019147da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T123827059217Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b7fb2c769f0b04517d16b93ce503b4a5f9f870f0bc69aa5ecd80ccb0e665c6d +size 5955602 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T123828360561Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T123828360561Z.json new file mode 100644 index 0000000000000000000000000000000000000000..77169cc11de7e9c59a8637b245a16c67d9b0ab5d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T123828360561Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111378091008, + "active_bytes_peak_delta": 23970744320, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111378091008, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110673447936, + "device": 2, + "inactive_split_bytes_current": 8595538944, + "inactive_split_bytes_peak": 11076187136, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116456947712, + "reserved_bytes_peak": 116456947712, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:38:28.360139Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110673447936, + "allocated_bytes_peak_delta": 23618422784, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87407346688, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87055025152, + "device": 2, + "inactive_split_bytes_current": 5424645120, + "inactive_split_bytes_peak": 8908559360, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 97498693632, + "reserved_bytes_peak": 97498693632, + "segment_count_current": 457, + "segment_count_peak": 457, + "timestamp_utc": "2026-08-01T12:38:27.198276Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8595538944, + "inactive_split_bytes_current_delta": 3170893824, + "inactive_split_bytes_peak": 11076187136, + "inactive_split_bytes_peak_delta": 2167627776, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116456947712, + "reserved_bytes_current_delta": 18958254080, + "reserved_bytes_peak": 116456947712, + "reserved_bytes_peak_delta": 18958254080, + "segment_count_current": 534, + "segment_count_current_delta": 77, + "segment_count_peak": 534, + "segment_count_peak_delta": 77, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T123828360561Z.pickle", + "timestamp_utc": "2026-08-01T12:38:28.476823Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T123828360561Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T123828360561Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f32a68b71a84a6d6f5909e9d787e9275e5ba5b50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T123828360561Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2eef0474be882e5225f7f9c48d8c3de06c9d02fd9aa9d48698412d404759e4a7 +size 6607513 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T123829611183Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T123829611183Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f0142164279b18b76ec9ed68ca1d6983d2a02048 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T123829611183Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111380889600, + "active_bytes_peak_delta": 2798592, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111380889600, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110676246528, + "device": 2, + "inactive_split_bytes_current": 8419378176, + "inactive_split_bytes_peak": 12583546880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117281128448, + "reserved_bytes_peak": 117281128448, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:38:29.610776Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110676246528, + "allocated_bytes_peak_delta": 2798592, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111378091008, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110673447936, + "device": 2, + "inactive_split_bytes_current": 8595538944, + "inactive_split_bytes_peak": 11076187136, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116456947712, + "reserved_bytes_peak": 116456947712, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:38:28.493188Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8419378176, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 12583546880, + "inactive_split_bytes_peak_delta": 1507359744, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117281128448, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117281128448, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 535, + "segment_count_current_delta": 1, + "segment_count_peak": 535, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T123829611183Z.pickle", + "timestamp_utc": "2026-08-01T12:38:29.723488Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T123829611183Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T123829611183Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2bb26b8122af6d5f00928bd3fda303a81132203f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T123829611183Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f08eb1174ce1528cc7c5a0837b589ccae9db9798b6c27cc1b05d60234d69314 +size 6098911 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T123831535968Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T123831535968Z.json new file mode 100644 index 0000000000000000000000000000000000000000..19c5e74b00bcaff2fd69e423fdf51950c3a75d39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T123831535968Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 167061993472, + "active_bytes_current_delta": 94554373120, + "active_bytes_peak": 167492779008, + "active_bytes_peak_delta": 56111889408, + "after": { + "active_bytes_current": 167061993472, + "active_bytes_peak": 167492779008, + "allocated_bytes_current": 109450319872, + "allocated_bytes_peak": 110684874240, + "device": 2, + "inactive_split_bytes_current": 10386425856, + "inactive_split_bytes_peak": 18425461248, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 182544498688, + "reserved_bytes_peak": 182544498688, + "segment_count_current": 847, + "segment_count_peak": 847, + "timestamp_utc": "2026-08-01T12:38:31.535305Z", + "world_size": 8 + }, + "allocated_bytes_current": 109450319872, + "allocated_bytes_current_delta": 36942699520, + "allocated_bytes_peak": 110684874240, + "allocated_bytes_peak_delta": 8627712, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111380889600, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110676246528, + "device": 2, + "inactive_split_bytes_current": 8419378176, + "inactive_split_bytes_peak": 12583546880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117281128448, + "reserved_bytes_peak": 117281128448, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:38:29.739591Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10386425856, + "inactive_split_bytes_current_delta": 1967047680, + "inactive_split_bytes_peak": 18425461248, + "inactive_split_bytes_peak_delta": 5841914368, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 182544498688, + "reserved_bytes_current_delta": 65263370240, + "reserved_bytes_peak": 182544498688, + "reserved_bytes_peak_delta": 65263370240, + "segment_count_current": 847, + "segment_count_current_delta": 312, + "segment_count_peak": 847, + "segment_count_peak_delta": 312, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T123831535968Z.pickle", + "timestamp_utc": "2026-08-01T12:38:31.673167Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T123831535968Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T123831535968Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f40f749fc4f55d37e65954be294a207f946cde70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T123831535968Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1ef7a6a095e8366195b735743bc64af82df841969f8992da3c6c2b27aaa0787 +size 5654499 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T123827059632Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T123827059632Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dab72a83222a9eeea1f7a98f988b7a437ebd0c85 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T123827059632Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 87407346688, + "active_bytes_peak_delta": 51941010432, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87407346688, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87055025152, + "device": 3, + "inactive_split_bytes_current": 5542085632, + "inactive_split_bytes_peak": 9808573440, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 97263812608, + "reserved_bytes_peak": 97263812608, + "segment_count_current": 456, + "segment_count_peak": 456, + "timestamp_utc": "2026-08-01T12:38:27.058939Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 87055025152, + "allocated_bytes_peak_delta": 51588688896, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:37:17.649100Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5542085632, + "inactive_split_bytes_current_delta": 5409266688, + "inactive_split_bytes_peak": 9808573440, + "inactive_split_bytes_peak_delta": 9648187392, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 97263812608, + "reserved_bytes_current_delta": 58019807232, + "reserved_bytes_peak": 97263812608, + "reserved_bytes_peak_delta": 58019807232, + "segment_count_current": 456, + "segment_count_current_delta": 385, + "segment_count_peak": 456, + "segment_count_peak_delta": 385, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T123827059632Z.pickle", + "timestamp_utc": "2026-08-01T12:38:27.180263Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T123827059632Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T123827059632Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..81920a39985867befa5b6d29d52c286303e75fb8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T123827059632Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ceae3d938e467033abf0a150e97e99c4ff04e6cfcc594945a1450a3cec0067c +size 5955726 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T123828360858Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T123828360858Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed7f0e5ddfcce3ceca73c1c5b66d9df9763fad94 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T123828360858Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111021753344, + "active_bytes_peak_delta": 23614406656, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111021753344, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110670362624, + "device": 3, + "inactive_split_bytes_current": 8421475328, + "inactive_split_bytes_peak": 11546929152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116819755008, + "reserved_bytes_peak": 116819755008, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T12:38:28.360366Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110670362624, + "allocated_bytes_peak_delta": 23615337472, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87407346688, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87055025152, + "device": 3, + "inactive_split_bytes_current": 5542085632, + "inactive_split_bytes_peak": 9808573440, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 97263812608, + "reserved_bytes_peak": 97263812608, + "segment_count_current": 456, + "segment_count_peak": 456, + "timestamp_utc": "2026-08-01T12:38:27.197616Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8421475328, + "inactive_split_bytes_current_delta": 2879389696, + "inactive_split_bytes_peak": 11546929152, + "inactive_split_bytes_peak_delta": 1738355712, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116819755008, + "reserved_bytes_current_delta": 19555942400, + "reserved_bytes_peak": 116819755008, + "reserved_bytes_peak_delta": 19555942400, + "segment_count_current": 539, + "segment_count_current_delta": 83, + "segment_count_peak": 539, + "segment_count_peak_delta": 83, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T123828360858Z.pickle", + "timestamp_utc": "2026-08-01T12:38:28.477174Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T123828360858Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T123828360858Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b778a10e7f108e64b4f6bd36884132bb0078e76d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T123828360858Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e61ee58301848666c5cda2ea0f3467c429b494cb941c72e5ca3c421decf6d0f9 +size 6086538 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T123829611414Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T123829611414Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3006df6eb5f3226f4cde7c21b008a40b43f7ca7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T123829611414Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111031189504, + "active_bytes_peak_delta": 9436160, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111031189504, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110678867968, + "device": 3, + "inactive_split_bytes_current": 8308229120, + "inactive_split_bytes_peak": 11546929152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116819755008, + "reserved_bytes_peak": 116819755008, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T12:38:29.610943Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110678867968, + "allocated_bytes_peak_delta": 8505344, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111021753344, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110670362624, + "device": 3, + "inactive_split_bytes_current": 8421475328, + "inactive_split_bytes_peak": 11546929152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116819755008, + "reserved_bytes_peak": 116819755008, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T12:38:28.491797Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8308229120, + "inactive_split_bytes_current_delta": -113246208, + "inactive_split_bytes_peak": 11546929152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116819755008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116819755008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 539, + "segment_count_current_delta": 0, + "segment_count_peak": 539, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T123829611414Z.pickle", + "timestamp_utc": "2026-08-01T12:38:29.725874Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T123829611414Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T123829611414Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7a78c782a9690bfee88b02aa3bd9ffa7a08d25fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T123829611414Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53f333d7483333e4410429dd7f85a627682f135da99fe5f7b7617d5119d0442f +size 6605502 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T123831564333Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T123831564333Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6d24d43045b0c1f22d4d23d7935cbfa1063420ef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T123831564333Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 167061469184, + "active_bytes_current_delta": 94553848832, + "active_bytes_peak": 167493303296, + "active_bytes_peak_delta": 56462113792, + "after": { + "active_bytes_current": 167061469184, + "active_bytes_peak": 167493303296, + "allocated_bytes_current": 109449795584, + "allocated_bytes_peak": 110685398528, + "device": 3, + "inactive_split_bytes_current": 9736833024, + "inactive_split_bytes_peak": 17303484928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 182710173696, + "reserved_bytes_peak": 182710173696, + "segment_count_current": 848, + "segment_count_peak": 848, + "timestamp_utc": "2026-08-01T12:38:31.563661Z", + "world_size": 8 + }, + "allocated_bytes_current": 109449795584, + "allocated_bytes_current_delta": 36942175232, + "allocated_bytes_peak": 110685398528, + "allocated_bytes_peak_delta": 6530560, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111031189504, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110678867968, + "device": 3, + "inactive_split_bytes_current": 8308229120, + "inactive_split_bytes_peak": 11546929152, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116819755008, + "reserved_bytes_peak": 116819755008, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T12:38:29.739791Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9736833024, + "inactive_split_bytes_current_delta": 1428603904, + "inactive_split_bytes_peak": 17303484928, + "inactive_split_bytes_peak_delta": 5756555776, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 182710173696, + "reserved_bytes_current_delta": 65890418688, + "reserved_bytes_peak": 182710173696, + "reserved_bytes_peak_delta": 65890418688, + "segment_count_current": 848, + "segment_count_current_delta": 309, + "segment_count_peak": 848, + "segment_count_peak_delta": 309, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T123831564333Z.pickle", + "timestamp_utc": "2026-08-01T12:38:31.695313Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T123831564333Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T123831564333Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..166d4cad268010283f668aed51eaf6dbc1a04ae3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T123831564333Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88a13494c4b125f7caebd10df70f34de979413082ccd645d199fc251a6d3a05c +size 5642732 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T123827059216Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T123827059216Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e5dc1db9a98a741d38d647036705d9cb686ff63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T123827059216Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509152256, + "active_bytes_current_delta": 37277254656, + "active_bytes_peak": 87409982464, + "active_bytes_peak_delta": 51943203840, + "after": { + "active_bytes_current": 72509152256, + "active_bytes_peak": 87409982464, + "allocated_bytes_current": 72509152256, + "allocated_bytes_peak": 87057660928, + "device": 4, + "inactive_split_bytes_current": 4890436608, + "inactive_split_bytes_peak": 9475339264, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 97026834432, + "reserved_bytes_peak": 97026834432, + "segment_count_current": 450, + "segment_count_peak": 450, + "timestamp_utc": "2026-08-01T12:38:27.058647Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509152256, + "allocated_bytes_current_delta": 37277254656, + "allocated_bytes_peak": 87057660928, + "allocated_bytes_peak_delta": 51590882304, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:37:17.649226Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4890436608, + "inactive_split_bytes_current_delta": 4758060032, + "inactive_split_bytes_peak": 9475339264, + "inactive_split_bytes_peak_delta": 9321929728, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 97026834432, + "reserved_bytes_current_delta": 57782829056, + "reserved_bytes_peak": 97026834432, + "reserved_bytes_peak_delta": 57782829056, + "segment_count_current": 450, + "segment_count_current_delta": 379, + "segment_count_peak": 450, + "segment_count_peak_delta": 379, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T123827059216Z.pickle", + "timestamp_utc": "2026-08-01T12:38:27.175640Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T123827059216Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T123827059216Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d223e414235f570f23349cef5f07a67f5b2190a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T123827059216Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:543a36650ecfa02f7673b7cf6554159a2dcf12fda615e4d8c45b0af718783a86 +size 5964233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T123828360543Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T123828360543Z.json new file mode 100644 index 0000000000000000000000000000000000000000..164629ffc7d53708ce3ba971c6758aeaf5e1e67d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T123828360543Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509152256, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111384130560, + "active_bytes_peak_delta": 23974148096, + "after": { + "active_bytes_current": 72509152256, + "active_bytes_peak": 111384130560, + "allocated_bytes_current": 72509152256, + "allocated_bytes_peak": 110681846272, + "device": 4, + "inactive_split_bytes_current": 8652727296, + "inactive_split_bytes_peak": 12063493120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116758937600, + "reserved_bytes_peak": 116758937600, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:38:28.360132Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509152256, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110681846272, + "allocated_bytes_peak_delta": 23624185344, + "before": { + "active_bytes_current": 72509152256, + "active_bytes_peak": 87409982464, + "allocated_bytes_current": 72509152256, + "allocated_bytes_peak": 87057660928, + "device": 4, + "inactive_split_bytes_current": 4890436608, + "inactive_split_bytes_peak": 9475339264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 97026834432, + "reserved_bytes_peak": 97026834432, + "segment_count_current": 450, + "segment_count_peak": 450, + "timestamp_utc": "2026-08-01T12:38:27.192499Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8652727296, + "inactive_split_bytes_current_delta": 3762290688, + "inactive_split_bytes_peak": 12063493120, + "inactive_split_bytes_peak_delta": 2588153856, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116758937600, + "reserved_bytes_current_delta": 19732103168, + "reserved_bytes_peak": 116758937600, + "reserved_bytes_peak_delta": 19732103168, + "segment_count_current": 534, + "segment_count_current_delta": 84, + "segment_count_peak": 534, + "segment_count_peak_delta": 84, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T123828360543Z.pickle", + "timestamp_utc": "2026-08-01T12:38:28.473318Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T123828360543Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T123828360543Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7a9dfef879535d4db30c9bf751474e4a8a54a9a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T123828360543Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c047bf8cb801fc424e021f7cc09b68fde899182f070da9ffc6a4cec5355824fa +size 6629966 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T123829611206Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T123829611206Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eb6fc99557e140cefe392ab32f3b262641b13603 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T123829611206Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509152256, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111388062208, + "active_bytes_peak_delta": 3931648, + "after": { + "active_bytes_current": 72509152256, + "active_bytes_peak": 111388062208, + "allocated_bytes_current": 72509152256, + "allocated_bytes_peak": 110683419136, + "device": 4, + "inactive_split_bytes_current": 8830985216, + "inactive_split_bytes_peak": 12063493120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116758937600, + "reserved_bytes_peak": 116758937600, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:38:29.610805Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509152256, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110683419136, + "allocated_bytes_peak_delta": 1572864, + "before": { + "active_bytes_current": 72509152256, + "active_bytes_peak": 111384130560, + "allocated_bytes_current": 72509152256, + "allocated_bytes_peak": 110681846272, + "device": 4, + "inactive_split_bytes_current": 8652727296, + "inactive_split_bytes_peak": 12063493120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116758937600, + "reserved_bytes_peak": 116758937600, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:38:28.487536Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8830985216, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 12063493120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116758937600, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116758937600, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 534, + "segment_count_current_delta": 0, + "segment_count_peak": 534, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T123829611206Z.pickle", + "timestamp_utc": "2026-08-01T12:38:29.720349Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T123829611206Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T123829611206Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9af3e107533ccabbe58df90488d67888ae904661 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T123829611206Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d793d3aea03aaeeefc0179917628f10a0b96822e11dec66f63e9ea2fcf2d32a +size 6625741 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T123831427998Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T123831427998Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c2465ac760645a030ee96d4ad977ad580fc62bcf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T123831427998Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 167419075584, + "active_bytes_current_delta": 94909923328, + "active_bytes_peak": 167419075584, + "active_bytes_peak_delta": 56031013376, + "after": { + "active_bytes_current": 167419075584, + "active_bytes_peak": 167419075584, + "allocated_bytes_current": 110159723520, + "allocated_bytes_peak": 110686144000, + "device": 4, + "inactive_split_bytes_current": 11323286528, + "inactive_split_bytes_peak": 19290315264, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 182882140160, + "reserved_bytes_peak": 182882140160, + "segment_count_current": 839, + "segment_count_peak": 839, + "timestamp_utc": "2026-08-01T12:38:31.427371Z", + "world_size": 8 + }, + "allocated_bytes_current": 110159723520, + "allocated_bytes_current_delta": 37650571264, + "allocated_bytes_peak": 110686144000, + "allocated_bytes_peak_delta": 2724864, + "before": { + "active_bytes_current": 72509152256, + "active_bytes_peak": 111388062208, + "allocated_bytes_current": 72509152256, + "allocated_bytes_peak": 110683419136, + "device": 4, + "inactive_split_bytes_current": 8830985216, + "inactive_split_bytes_peak": 12063493120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116758937600, + "reserved_bytes_peak": 116758937600, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:38:29.734456Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11323286528, + "inactive_split_bytes_current_delta": 2492301312, + "inactive_split_bytes_peak": 19290315264, + "inactive_split_bytes_peak_delta": 7226822144, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 182882140160, + "reserved_bytes_current_delta": 66123202560, + "reserved_bytes_peak": 182882140160, + "reserved_bytes_peak_delta": 66123202560, + "segment_count_current": 839, + "segment_count_current_delta": 305, + "segment_count_peak": 839, + "segment_count_peak_delta": 305, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T123831427998Z.pickle", + "timestamp_utc": "2026-08-01T12:38:31.552166Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T123831427998Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T123831427998Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f022f9f6b03e0fd2ac7a94b9ffd2ab352ed1b816 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T123831427998Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db4acdaf1e4ec34a772668e7dab02c48b9e16bd702befa912f99b11a89fb9b66 +size 6395890 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T123827059280Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T123827059280Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a93c8d5f24912fa1e8fc41a73018327b6ed0c55b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T123827059280Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507882496, + "active_bytes_current_delta": 37276427264, + "active_bytes_peak": 87407346688, + "active_bytes_peak_delta": 51941010432, + "after": { + "active_bytes_current": 72507882496, + "active_bytes_peak": 87407346688, + "allocated_bytes_current": 72507882496, + "allocated_bytes_peak": 87055025152, + "device": 5, + "inactive_split_bytes_current": 5306942464, + "inactive_split_bytes_peak": 9565361152, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 97263812608, + "reserved_bytes_peak": 97263812608, + "segment_count_current": 456, + "segment_count_peak": 456, + "timestamp_utc": "2026-08-01T12:38:27.058657Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507882496, + "allocated_bytes_current_delta": 37276427264, + "allocated_bytes_peak": 87055025152, + "allocated_bytes_peak_delta": 51588688896, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:37:17.648653Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5306942464, + "inactive_split_bytes_current_delta": 5174123520, + "inactive_split_bytes_peak": 9565361152, + "inactive_split_bytes_peak_delta": 9404975104, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 97263812608, + "reserved_bytes_current_delta": 58019807232, + "reserved_bytes_peak": 97263812608, + "reserved_bytes_peak_delta": 58019807232, + "segment_count_current": 456, + "segment_count_current_delta": 385, + "segment_count_peak": 456, + "segment_count_peak_delta": 385, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T123827059280Z.pickle", + "timestamp_utc": "2026-08-01T12:38:27.175429Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T123827059280Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T123827059280Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8199805ea096ef46088ffdcd54287f8b82055f91 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T123827059280Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6b3d1f4cffbc474a56805fa0585fbc5d31cbf883d3d6d539d480f81564c486a +size 5955355 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T123828360531Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T123828360531Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d2639dbce1d190d80f74d7a0fbad1aeb69624841 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T123828360531Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507882496, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111374902272, + "active_bytes_peak_delta": 23967555584, + "after": { + "active_bytes_current": 72507882496, + "active_bytes_peak": 111374902272, + "allocated_bytes_current": 72507882496, + "allocated_bytes_peak": 110675722240, + "device": 5, + "inactive_split_bytes_current": 8773534720, + "inactive_split_bytes_peak": 11319871488, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116876378112, + "reserved_bytes_peak": 116876378112, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:38:28.360120Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507882496, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110675722240, + "allocated_bytes_peak_delta": 23620697088, + "before": { + "active_bytes_current": 72507882496, + "active_bytes_peak": 87407346688, + "allocated_bytes_current": 72507882496, + "allocated_bytes_peak": 87055025152, + "device": 5, + "inactive_split_bytes_current": 5306942464, + "inactive_split_bytes_peak": 9565361152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 97263812608, + "reserved_bytes_peak": 97263812608, + "segment_count_current": 456, + "segment_count_peak": 456, + "timestamp_utc": "2026-08-01T12:38:27.192378Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8773534720, + "inactive_split_bytes_current_delta": 3466592256, + "inactive_split_bytes_peak": 11319871488, + "inactive_split_bytes_peak_delta": 1754510336, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116876378112, + "reserved_bytes_current_delta": 19612565504, + "reserved_bytes_peak": 116876378112, + "reserved_bytes_peak_delta": 19612565504, + "segment_count_current": 536, + "segment_count_current_delta": 80, + "segment_count_peak": 536, + "segment_count_peak_delta": 80, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T123828360531Z.pickle", + "timestamp_utc": "2026-08-01T12:38:28.471494Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T123828360531Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T123828360531Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b9c266f4e759e8e2ad82c622d2d0ded1473e331a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T123828360531Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb837e6e5a572d471b11ad4510949b3fd23dbd0f9bbd73a0d6937f15d1493458 +size 6085578 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T123829611166Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T123829611166Z.json new file mode 100644 index 0000000000000000000000000000000000000000..45a51be9578a3731c11996356b3a81baf05ee435 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T123829611166Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507882496, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111380788224, + "active_bytes_peak_delta": 5885952, + "after": { + "active_bytes_current": 72507882496, + "active_bytes_peak": 111380788224, + "allocated_bytes_current": 72507882496, + "allocated_bytes_peak": 110676145152, + "device": 5, + "inactive_split_bytes_current": 9121661952, + "inactive_split_bytes_peak": 11767252992, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116876378112, + "reserved_bytes_peak": 116876378112, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:38:29.610772Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507882496, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110676145152, + "allocated_bytes_peak_delta": 422912, + "before": { + "active_bytes_current": 72507882496, + "active_bytes_peak": 111374902272, + "allocated_bytes_current": 72507882496, + "allocated_bytes_peak": 110675722240, + "device": 5, + "inactive_split_bytes_current": 8773534720, + "inactive_split_bytes_peak": 11319871488, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116876378112, + "reserved_bytes_peak": 116876378112, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:38:28.485434Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9121661952, + "inactive_split_bytes_current_delta": 348127232, + "inactive_split_bytes_peak": 11767252992, + "inactive_split_bytes_peak_delta": 447381504, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116876378112, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116876378112, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T123829611166Z.pickle", + "timestamp_utc": "2026-08-01T12:38:29.719522Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T123829611166Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T123829611166Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..39672738050e5fcb12f6a9a38e7d0602ceab1b16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T123829611166Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1796ca0b9e09e0644608b52101e7f29bbf9aa52944876141e43e4d73ca07fe0f +size 6604411 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T123831514065Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T123831514065Z.json new file mode 100644 index 0000000000000000000000000000000000000000..682e349bd802f714c07ef4fd04e73ae0add1e76b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T123831514065Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 166700055552, + "active_bytes_current_delta": 94192173056, + "active_bytes_peak": 166743047168, + "active_bytes_peak_delta": 55362258944, + "after": { + "active_bytes_current": 166700055552, + "active_bytes_peak": 166743047168, + "allocated_bytes_current": 109440703488, + "allocated_bytes_peak": 110685660672, + "device": 5, + "inactive_split_bytes_current": 11088102400, + "inactive_split_bytes_peak": 18469239296, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 182810836992, + "reserved_bytes_peak": 182810836992, + "segment_count_current": 843, + "segment_count_peak": 843, + "timestamp_utc": "2026-08-01T12:38:31.513377Z", + "world_size": 8 + }, + "allocated_bytes_current": 109440703488, + "allocated_bytes_current_delta": 36932820992, + "allocated_bytes_peak": 110685660672, + "allocated_bytes_peak_delta": 9515520, + "before": { + "active_bytes_current": 72507882496, + "active_bytes_peak": 111380788224, + "allocated_bytes_current": 72507882496, + "allocated_bytes_peak": 110676145152, + "device": 5, + "inactive_split_bytes_current": 9121661952, + "inactive_split_bytes_peak": 11767252992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116876378112, + "reserved_bytes_peak": 116876378112, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:38:29.733137Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11088102400, + "inactive_split_bytes_current_delta": 1966440448, + "inactive_split_bytes_peak": 18469239296, + "inactive_split_bytes_peak_delta": 6701986304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 182810836992, + "reserved_bytes_current_delta": 65934458880, + "reserved_bytes_peak": 182810836992, + "reserved_bytes_peak_delta": 65934458880, + "segment_count_current": 843, + "segment_count_current_delta": 307, + "segment_count_peak": 843, + "segment_count_peak_delta": 307, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T123831514065Z.pickle", + "timestamp_utc": "2026-08-01T12:38:31.643770Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T123831514065Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T123831514065Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5362c72b6558fce3e4aee975cb719b07b662ec48 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T123831514065Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1005135ad499a24647a7fab28fdf899cf41f157f40cf5fd3d416feed326ddce0 +size 5617989 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T123827059278Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T123827059278Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0073e28306dce0753b2e8d7586220ca10169cfe7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T123827059278Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 87408395776, + "active_bytes_peak_delta": 51942059520, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87408395776, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87056074240, + "device": 6, + "inactive_split_bytes_current": 5776966656, + "inactive_split_bytes_peak": 9044989952, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 97733574656, + "reserved_bytes_peak": 97733574656, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T12:38:27.058662Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 87056074240, + "allocated_bytes_peak_delta": 51589737984, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:37:17.649116Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5776966656, + "inactive_split_bytes_current_delta": 5644147712, + "inactive_split_bytes_peak": 9044989952, + "inactive_split_bytes_peak_delta": 8884603904, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 97733574656, + "reserved_bytes_current_delta": 58489569280, + "reserved_bytes_peak": 97733574656, + "reserved_bytes_peak_delta": 58489569280, + "segment_count_current": 458, + "segment_count_current_delta": 387, + "segment_count_peak": 458, + "segment_count_peak_delta": 387, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T123827059278Z.pickle", + "timestamp_utc": "2026-08-01T12:38:27.178262Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T123827059278Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T123827059278Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..35297c6a0c1895c9af868057813a3cb4b30509f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T123827059278Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64344d993ca38b26d0cb4104f8463746ef2c2d9e93c12cd7b9f2cb5e0c582536 +size 5955777 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T123828360597Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T123828360597Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3adb239166dc722ea8ec04ea60569260b2a47858 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T123828360597Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111034073088, + "active_bytes_peak_delta": 23625677312, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111034073088, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110681751552, + "device": 6, + "inactive_split_bytes_current": 9300182016, + "inactive_split_bytes_peak": 10705758208, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116398227456, + "reserved_bytes_peak": 116398227456, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T12:38:28.360181Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110681751552, + "allocated_bytes_peak_delta": 23625677312, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 87408395776, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 87056074240, + "device": 6, + "inactive_split_bytes_current": 5776966656, + "inactive_split_bytes_peak": 9044989952, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 97733574656, + "reserved_bytes_peak": 97733574656, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T12:38:27.199033Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9300182016, + "inactive_split_bytes_current_delta": 3523215360, + "inactive_split_bytes_peak": 10705758208, + "inactive_split_bytes_peak_delta": 1660768256, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116398227456, + "reserved_bytes_current_delta": 18664652800, + "reserved_bytes_peak": 116398227456, + "reserved_bytes_peak_delta": 18664652800, + "segment_count_current": 533, + "segment_count_current_delta": 75, + "segment_count_peak": 533, + "segment_count_peak_delta": 75, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T123828360597Z.pickle", + "timestamp_utc": "2026-08-01T12:38:28.472428Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T123828360597Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T123828360597Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7d94477dc999f3126ca538952e71bd88eddc70fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T123828360597Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bdf0287d45ef2137a4e49e1dcc0d4aab54a71ef6eabc1d79f64783dc64a45db +size 6085250 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T123829611208Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T123829611208Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8de625690388ce69fa0983107a6437093603cc2a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T123829611208Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111380712448, + "active_bytes_peak_delta": 346639360, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111380712448, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110681751552, + "device": 6, + "inactive_split_bytes_current": 7775552512, + "inactive_split_bytes_peak": 10879207424, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117222408192, + "reserved_bytes_peak": 117222408192, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:38:29.610802Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110681751552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111034073088, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110681751552, + "device": 6, + "inactive_split_bytes_current": 9300182016, + "inactive_split_bytes_peak": 10705758208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116398227456, + "reserved_bytes_peak": 116398227456, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T12:38:28.488832Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7775552512, + "inactive_split_bytes_current_delta": -1524629504, + "inactive_split_bytes_peak": 10879207424, + "inactive_split_bytes_peak_delta": 173449216, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117222408192, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117222408192, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 534, + "segment_count_current_delta": 1, + "segment_count_peak": 534, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T123829611208Z.pickle", + "timestamp_utc": "2026-08-01T12:38:29.723250Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T123829611208Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T123829611208Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..108685c1ef003f56653bd011e66a19a673a92678 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T123829611208Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab27720026b0bfe031c9976e3acbef7a4ed447a6827ec2d26b09bd93cb7edaa3 +size 6098677 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T123831539340Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T123831539340Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f0d1288fd23528231c7a227f0eaadb5ff3b1b7cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T123831539340Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 167469627392, + "active_bytes_current_delta": 94962007040, + "active_bytes_peak": 167487536128, + "active_bytes_peak_delta": 56106823680, + "after": { + "active_bytes_current": 167469627392, + "active_bytes_peak": 167487536128, + "allocated_bytes_current": 109857953792, + "allocated_bytes_peak": 110682514944, + "device": 6, + "inactive_split_bytes_current": 10165438464, + "inactive_split_bytes_peak": 17949669888, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 182540304384, + "reserved_bytes_peak": 182540304384, + "segment_count_current": 850, + "segment_count_peak": 850, + "timestamp_utc": "2026-08-01T12:38:31.538656Z", + "world_size": 8 + }, + "allocated_bytes_current": 109857953792, + "allocated_bytes_current_delta": 37350333440, + "allocated_bytes_peak": 110682514944, + "allocated_bytes_peak_delta": 763392, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 111380712448, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 110681751552, + "device": 6, + "inactive_split_bytes_current": 7775552512, + "inactive_split_bytes_peak": 10879207424, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117222408192, + "reserved_bytes_peak": 117222408192, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:38:29.739029Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10165438464, + "inactive_split_bytes_current_delta": 2389885952, + "inactive_split_bytes_peak": 17949669888, + "inactive_split_bytes_peak_delta": 7070462464, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 182540304384, + "reserved_bytes_current_delta": 65317896192, + "reserved_bytes_peak": 182540304384, + "reserved_bytes_peak_delta": 65317896192, + "segment_count_current": 850, + "segment_count_current_delta": 316, + "segment_count_peak": 850, + "segment_count_peak_delta": 316, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T123831539340Z.pickle", + "timestamp_utc": "2026-08-01T12:38:31.676222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T123831539340Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T123831539340Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f56006bf0f0826646ffb0858a5ebfb27443dd391 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T123831539340Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96ec7e5df9a4414934fead67b6a7ed1045eba8fe93583f7e2271920da3f5f88b +size 5813087 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T123827059216Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T123827059216Z.json new file mode 100644 index 0000000000000000000000000000000000000000..02b874382dbb479b825a93b5ea0be83e7f86f856 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T123827059216Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 37276165632, + "active_bytes_peak": 87407346688, + "active_bytes_peak_delta": 51941010432, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 87407346688, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 87055025152, + "device": 7, + "inactive_split_bytes_current": 4896162304, + "inactive_split_bytes_peak": 9218225152, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 97028931584, + "reserved_bytes_peak": 97028931584, + "segment_count_current": 456, + "segment_count_peak": 456, + "timestamp_utc": "2026-08-01T12:38:27.058661Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 37276165632, + "allocated_bytes_peak": 87055025152, + "allocated_bytes_peak_delta": 51588688896, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:37:17.647979Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4896162304, + "inactive_split_bytes_current_delta": 4763343360, + "inactive_split_bytes_peak": 9218225152, + "inactive_split_bytes_peak_delta": 9057839104, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 97028931584, + "reserved_bytes_current_delta": 57784926208, + "reserved_bytes_peak": 97028931584, + "reserved_bytes_peak_delta": 57784926208, + "segment_count_current": 456, + "segment_count_current_delta": 385, + "segment_count_peak": 456, + "segment_count_peak_delta": 385, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T123827059216Z.pickle", + "timestamp_utc": "2026-08-01T12:38:27.175502Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T123827059216Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T123827059216Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fcdce5978c4c8fa5edb9e193a59992a55d8611d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T123827059216Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58747d72e5287f622e20cee438ea38d1d313d12a7774f695ff6513e53954fc6b +size 5956878 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T123828360604Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T123828360604Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6d24c4d5dc1f6ce35994c578eecc6b346c12627 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T123828360604Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111375932928, + "active_bytes_peak_delta": 23968586240, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 111375932928, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 110677012992, + "device": 7, + "inactive_split_bytes_current": 8247411200, + "inactive_split_bytes_peak": 12484980224, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116456947712, + "reserved_bytes_peak": 116456947712, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:38:28.360201Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110677012992, + "allocated_bytes_peak_delta": 23621987840, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 87407346688, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 87055025152, + "device": 7, + "inactive_split_bytes_current": 4896162304, + "inactive_split_bytes_peak": 9218225152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 97028931584, + "reserved_bytes_peak": 97028931584, + "segment_count_current": 456, + "segment_count_peak": 456, + "timestamp_utc": "2026-08-01T12:38:27.192324Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8247411200, + "inactive_split_bytes_current_delta": 3351248896, + "inactive_split_bytes_peak": 12484980224, + "inactive_split_bytes_peak_delta": 3266755072, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116456947712, + "reserved_bytes_current_delta": 19428016128, + "reserved_bytes_peak": 116456947712, + "reserved_bytes_peak_delta": 19428016128, + "segment_count_current": 534, + "segment_count_current_delta": 78, + "segment_count_peak": 534, + "segment_count_peak_delta": 78, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T123828360604Z.pickle", + "timestamp_utc": "2026-08-01T12:38:28.474538Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T123828360604Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T123828360604Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d0c092a8754796790eb5275c021b623bc8b13067 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T123828360604Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:450ee3d1598cf17fa0fb1970aacf15f7bade6fb59a082348727bfe12af0e7afb +size 6087414 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T123829611254Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T123829611254Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e77c80d0d417b887d6a036f1ec9195b4d500423a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T123829611254Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111384759296, + "active_bytes_peak_delta": 8826368, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 111384759296, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 110680235008, + "device": 7, + "inactive_split_bytes_current": 9069494784, + "inactive_split_bytes_peak": 12484980224, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117281128448, + "reserved_bytes_peak": 117281128448, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:38:29.610852Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110680235008, + "allocated_bytes_peak_delta": 3222016, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 111375932928, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 110677012992, + "device": 7, + "inactive_split_bytes_current": 8247411200, + "inactive_split_bytes_peak": 12484980224, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116456947712, + "reserved_bytes_peak": 116456947712, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:38:28.488701Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9069494784, + "inactive_split_bytes_current_delta": 822083584, + "inactive_split_bytes_peak": 12484980224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117281128448, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117281128448, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 535, + "segment_count_current_delta": 1, + "segment_count_peak": 535, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T123829611254Z.pickle", + "timestamp_utc": "2026-08-01T12:38:29.719016Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T123829611254Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T123829611254Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..66fa25bf04bd156ee179793e2658f2a7dcfe73bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T123829611254Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f83de64c164756efe4c6764881df5f836c7f7b61354a07be1c2d0e592621673 +size 6101158 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T123831266989Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T123831266989Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e9e10234be8726656f55d44b5ccb0e83d66d1727 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T123831266989Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 166699793920, + "active_bytes_current_delta": 94192173056, + "active_bytes_peak": 166742785536, + "active_bytes_peak_delta": 55358026240, + "after": { + "active_bytes_current": 166699793920, + "active_bytes_peak": 166742785536, + "allocated_bytes_current": 109440441856, + "allocated_bytes_peak": 110685399040, + "device": 7, + "inactive_split_bytes_current": 11505697280, + "inactive_split_bytes_peak": 19071383552, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 182758408192, + "reserved_bytes_peak": 182758408192, + "segment_count_current": 842, + "segment_count_peak": 842, + "timestamp_utc": "2026-08-01T12:38:31.266291Z", + "world_size": 8 + }, + "allocated_bytes_current": 109440441856, + "allocated_bytes_current_delta": 36932820992, + "allocated_bytes_peak": 110685399040, + "allocated_bytes_peak_delta": 5164032, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 111384759296, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 110680235008, + "device": 7, + "inactive_split_bytes_current": 9069494784, + "inactive_split_bytes_peak": 12484980224, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117281128448, + "reserved_bytes_peak": 117281128448, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:38:29.732518Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11505697280, + "inactive_split_bytes_current_delta": 2436202496, + "inactive_split_bytes_peak": 19071383552, + "inactive_split_bytes_peak_delta": 6586403328, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 182758408192, + "reserved_bytes_current_delta": 65477279744, + "reserved_bytes_peak": 182758408192, + "reserved_bytes_peak_delta": 65477279744, + "segment_count_current": 842, + "segment_count_current_delta": 307, + "segment_count_peak": 842, + "segment_count_peak_delta": 307, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T123831266989Z.pickle", + "timestamp_utc": "2026-08-01T12:38:31.399887Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T123831266989Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T123831266989Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e287ae45076740d3fbcafbc1db349bb355fcfa1b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T123831266989Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a9151281cfbff8a4eb19f1de3aec908d7f6ae8db454e49cc4120a18158cda25 +size 5645467 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..4c988f85ec75f78260076e9b94a1928c4985fa00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 4096 +capacity: 1.25 +expert_dispatch_overlap: off +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6016 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 241 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T124043404332Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T124043404332Z.json new file mode 100644 index 0000000000000000000000000000000000000000..931f15306ed1ef1eb37ade0e647526917ddce08b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T124043404332Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922511360, + "active_bytes_current_delta": 31691056128, + "active_bytes_peak": 89676963328, + "active_bytes_peak_delta": 54210627072, + "after": { + "active_bytes_current": 66922511360, + "active_bytes_peak": 89676963328, + "allocated_bytes_current": 66922511360, + "allocated_bytes_peak": 89676963328, + "device": 0, + "inactive_split_bytes_current": 8767898624, + "inactive_split_bytes_peak": 10469435392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 95546245120, + "reserved_bytes_peak": 95546245120, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.403850Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922511360, + "allocated_bytes_current_delta": 31691056128, + "allocated_bytes_peak": 89676963328, + "allocated_bytes_peak_delta": 54210627072, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:39:34.005316Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8767898624, + "inactive_split_bytes_current_delta": 8632982528, + "inactive_split_bytes_peak": 10469435392, + "inactive_split_bytes_peak_delta": 10308803584, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 95546245120, + "reserved_bytes_current_delta": 56300142592, + "reserved_bytes_peak": 95546245120, + "reserved_bytes_peak_delta": 56300142592, + "segment_count_current": 467, + "segment_count_current_delta": 395, + "segment_count_peak": 467, + "segment_count_peak_delta": 395, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T124043404332Z.pickle", + "timestamp_utc": "2026-08-01T12:40:43.516433Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T124043404332Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T124043404332Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..75a583678d83581e559b079386a550b1ae187e5a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T124043404332Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9c4ca6d355af6732a2979936372929ec3e890ebe46bf9a2b58fd3ca092501c3 +size 5391229 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T124044833992Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T124044833992Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4062e504dd584da08a04f8662fc7b6d128a18026 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T124044833992Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922511360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113303809024, + "active_bytes_peak_delta": 23626845696, + "after": { + "active_bytes_current": 66922511360, + "active_bytes_peak": 113303809024, + "allocated_bytes_current": 66922511360, + "allocated_bytes_peak": 113303809024, + "device": 0, + "inactive_split_bytes_current": 10565157888, + "inactive_split_bytes_peak": 10565157888, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116085751808, + "reserved_bytes_peak": 116085751808, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:40:44.833572Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922511360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113303809024, + "allocated_bytes_peak_delta": 23626845696, + "before": { + "active_bytes_current": 66922511360, + "active_bytes_peak": 89676963328, + "allocated_bytes_current": 66922511360, + "allocated_bytes_peak": 89676963328, + "device": 0, + "inactive_split_bytes_current": 8767898624, + "inactive_split_bytes_peak": 10469435392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 95546245120, + "reserved_bytes_peak": 95546245120, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.536087Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10565157888, + "inactive_split_bytes_current_delta": 1797259264, + "inactive_split_bytes_peak": 10565157888, + "inactive_split_bytes_peak_delta": 95722496, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116085751808, + "reserved_bytes_current_delta": 20539506688, + "reserved_bytes_peak": 116085751808, + "reserved_bytes_peak_delta": 20539506688, + "segment_count_current": 541, + "segment_count_current_delta": 74, + "segment_count_peak": 541, + "segment_count_peak_delta": 74, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T124044833992Z.pickle", + "timestamp_utc": "2026-08-01T12:40:44.939865Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T124044833992Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T124044833992Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ddb554b06868bfaa8d9c9d8e48ba123486dffdb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T124044833992Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af236eb7be3f0026bc926e86d25fa19714e2a73179d676a5aae48363aafa54ef +size 5570951 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T124045966999Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T124045966999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9e7e32611ff86963e9983aabeafbff85b15477ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T124045966999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922511360, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113305342464, + "active_bytes_peak_delta": 1533440, + "after": { + "active_bytes_current": 66922511360, + "active_bytes_peak": 113305342464, + "allocated_bytes_current": 66922511360, + "allocated_bytes_peak": 113305342464, + "device": 0, + "inactive_split_bytes_current": 10045064192, + "inactive_split_bytes_peak": 10759481344, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116085751808, + "reserved_bytes_peak": 116085751808, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:40:45.966580Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922511360, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113305342464, + "allocated_bytes_peak_delta": 1533440, + "before": { + "active_bytes_current": 66922511360, + "active_bytes_peak": 113303809024, + "allocated_bytes_current": 66922511360, + "allocated_bytes_peak": 113303809024, + "device": 0, + "inactive_split_bytes_current": 10565157888, + "inactive_split_bytes_peak": 10565157888, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116085751808, + "reserved_bytes_peak": 116085751808, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:40:44.953533Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10045064192, + "inactive_split_bytes_current_delta": -520093696, + "inactive_split_bytes_peak": 10759481344, + "inactive_split_bytes_peak_delta": 194323456, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116085751808, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116085751808, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 541, + "segment_count_current_delta": 0, + "segment_count_peak": 541, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T124045966999Z.pickle", + "timestamp_utc": "2026-08-01T12:40:46.066949Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T124045966999Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T124045966999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a09d9ac215009498da18bda5dc8d600da34b0fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T124045966999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cc289bbab0527e94724f8695c9904db67af9c1c9a0fa6d14abb74c7d036f38a +size 5572776 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T124048704729Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T124048704729Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7ec90f8ab5748812705343af24c8d7572ceb89aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T124048704729Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939554816, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131477284864, + "active_bytes_peak_delta": 18171942400, + "after": { + "active_bytes_current": 66939554816, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66939554816, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_peak": 18182715392, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_peak": 150011379712, + "segment_count_current": 766, + "segment_count_peak": 766, + "timestamp_utc": "2026-08-01T12:40:48.704146Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939554816, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113326147072, + "allocated_bytes_peak_delta": 20804608, + "before": { + "active_bytes_current": 66922511360, + "active_bytes_peak": 113305342464, + "allocated_bytes_current": 66922511360, + "allocated_bytes_peak": 113305342464, + "device": 0, + "inactive_split_bytes_current": 10045064192, + "inactive_split_bytes_peak": 10759481344, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116085751808, + "reserved_bytes_peak": 116085751808, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T12:40:46.079658Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_current_delta": 2040262656, + "inactive_split_bytes_peak": 18182715392, + "inactive_split_bytes_peak_delta": 7423234048, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_current_delta": 33925627904, + "reserved_bytes_peak": 150011379712, + "reserved_bytes_peak_delta": 33925627904, + "segment_count_current": 766, + "segment_count_current_delta": 225, + "segment_count_peak": 766, + "segment_count_peak_delta": 225, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T124048704729Z.pickle", + "timestamp_utc": "2026-08-01T12:40:48.822279Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T124048704729Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T124048704729Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c01c07298a22214d4b777b9b6782226b71724ee0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T124048704729Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9399298eaa41d47150d1eb0cbc3fa973f1f38d53f74aec7dd7d94f24bbd51232 +size 5868901 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T124049788512Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T124049788512Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d80d699c4e03957aa4405d0bdff8bab0639b818 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T124049788512Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939554816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131477284864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939554816, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66939554816, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_peak": 18182715392, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_peak": 150011379712, + "segment_count_current": 766, + "segment_count_peak": 766, + "timestamp_utc": "2026-08-01T12:40:49.788037Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939554816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113326147072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939554816, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66939554816, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_peak": 18182715392, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_peak": 150011379712, + "segment_count_current": 766, + "segment_count_peak": 766, + "timestamp_utc": "2026-08-01T12:40:48.836999Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 18182715392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150011379712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 766, + "segment_count_current_delta": 0, + "segment_count_peak": 766, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T124049788512Z.pickle", + "timestamp_utc": "2026-08-01T12:40:49.795278Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T124049788512Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T124049788512Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..66eef7534d6168194bfb2cbcd5ce3027f764a494 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T124049788512Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad127ab0f835b365c10e29bdaad5172004b44746f8db746a3a7157085f01333e +size 230035 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T124050737638Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T124050737638Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1daa9dc8b5040b1d2b6b3dbe14fc8ff5d2623e25 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T124050737638Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939554816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131477284864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939554816, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66939554816, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_peak": 18182715392, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_peak": 150011379712, + "segment_count_current": 766, + "segment_count_peak": 766, + "timestamp_utc": "2026-08-01T12:40:50.737234Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939554816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113326147072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939554816, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66939554816, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_peak": 18182715392, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_peak": 150011379712, + "segment_count_current": 766, + "segment_count_peak": 766, + "timestamp_utc": "2026-08-01T12:40:49.796239Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 18182715392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150011379712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 766, + "segment_count_current_delta": 0, + "segment_count_peak": 766, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T124050737638Z.pickle", + "timestamp_utc": "2026-08-01T12:40:50.744063Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T124050737638Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T124050737638Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4125d5b4b41cfed156fc43dd5b829ec6c468b256 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T124050737638Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:807420b120ee80484117cd08f819184c76784088ef7927be12f6d2e891b1565e +size 230035 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T124051686785Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T124051686785Z.json new file mode 100644 index 0000000000000000000000000000000000000000..243e4d6303ba8c9f315c55c6015a74d810d851a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T124051686785Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939554816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131477284864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939554816, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66939554816, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_peak": 18182715392, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_peak": 150011379712, + "segment_count_current": 766, + "segment_count_peak": 766, + "timestamp_utc": "2026-08-01T12:40:51.686362Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939554816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113326147072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939554816, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66939554816, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_peak": 18182715392, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_peak": 150011379712, + "segment_count_current": 766, + "segment_count_peak": 766, + "timestamp_utc": "2026-08-01T12:40:50.744992Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 18182715392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150011379712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 766, + "segment_count_current_delta": 0, + "segment_count_peak": 766, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T124051686785Z.pickle", + "timestamp_utc": "2026-08-01T12:40:51.693398Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T124051686785Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T124051686785Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f780895bf46bfeba6994c7353133d5aa0ca084d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T124051686785Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:160d978ec5dd3249e5850c03cb26c69f9f56f594dedd7c75a6e855f3168a76d4 +size 230035 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T124052629431Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T124052629431Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ccc80f9ef435191648871a64bc27da741dd7f74 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T124052629431Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939554816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131477284864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939554816, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66939554816, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_peak": 18182715392, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_peak": 150011379712, + "segment_count_current": 766, + "segment_count_peak": 766, + "timestamp_utc": "2026-08-01T12:40:52.629026Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939554816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113326147072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939554816, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66939554816, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_peak": 18182715392, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_peak": 150011379712, + "segment_count_current": 766, + "segment_count_peak": 766, + "timestamp_utc": "2026-08-01T12:40:51.694410Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 18182715392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150011379712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 766, + "segment_count_current_delta": 0, + "segment_count_peak": 766, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T124052629431Z.pickle", + "timestamp_utc": "2026-08-01T12:40:52.635524Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T124052629431Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T124052629431Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ddf54e1d62b05885839831177781f8f30b84e69 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T124052629431Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa5b1f8a51d5543fe17852e792448741ce8439b340b13304de50f4a637198b4e +size 230035 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T124053569204Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T124053569204Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ffdf893cbc6131750595a71961507401ad97f1c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T124053569204Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939554816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131477284864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939554816, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66939554816, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_peak": 18182715392, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_peak": 150011379712, + "segment_count_current": 766, + "segment_count_peak": 766, + "timestamp_utc": "2026-08-01T12:40:53.568772Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939554816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113326147072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939554816, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66939554816, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_peak": 18182715392, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_peak": 150011379712, + "segment_count_current": 766, + "segment_count_peak": 766, + "timestamp_utc": "2026-08-01T12:40:52.636467Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 18182715392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150011379712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 766, + "segment_count_current_delta": 0, + "segment_count_peak": 766, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T124053569204Z.pickle", + "timestamp_utc": "2026-08-01T12:40:53.575831Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T124053569204Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T124053569204Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fa72f551044bceb38092f101f47dcccdc7f306c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T124053569204Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46cc256a91c7ec4db4ed604b2c8d5e47d9de9efda58c2ed033d75c07641cb247 +size 230035 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T124153083492Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T124153083492Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e4e3d8358d136c6189c26f20bdcca1509c85341 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T124153083492Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66956481536, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 131477284864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66956481536, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66956481536, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 7899261952, + "inactive_split_bytes_peak": 18182715392, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150080585728, + "reserved_bytes_peak": 150080585728, + "segment_count_current": 771, + "segment_count_peak": 771, + "timestamp_utc": "2026-08-01T12:41:53.082990Z", + "world_size": 8 + }, + "allocated_bytes_current": 66956481536, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 113326147072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939554816, + "active_bytes_peak": 131477284864, + "allocated_bytes_current": 66939554816, + "allocated_bytes_peak": 113326147072, + "device": 0, + "inactive_split_bytes_current": 12085326848, + "inactive_split_bytes_peak": 18182715392, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 150011379712, + "reserved_bytes_peak": 150011379712, + "segment_count_current": 766, + "segment_count_peak": 766, + "timestamp_utc": "2026-08-01T12:40:53.576795Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7899261952, + "inactive_split_bytes_current_delta": -4186064896, + "inactive_split_bytes_peak": 18182715392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150080585728, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 150080585728, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 771, + "segment_count_current_delta": 5, + "segment_count_peak": 771, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T124153083492Z.pickle", + "timestamp_utc": "2026-08-01T12:41:53.166031Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T124153083492Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T124153083492Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea52c538b2c85c6e907684e07b2296cd86a80844 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T124153083492Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:724cfcb078803ce2fa15c10eced8f7ca7b010804334b0e088ff121bbe3c8ab14 +size 3980015 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T124043404331Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T124043404331Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6198e10a4f5557344c1b94d6e35e1f09672fbde2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T124043404331Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 89675354112, + "active_bytes_peak_delta": 54209017856, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89675354112, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89675354112, + "device": 1, + "inactive_split_bytes_current": 8437941248, + "inactive_split_bytes_peak": 9494840320, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 95693045760, + "reserved_bytes_peak": 95693045760, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.403845Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 89675354112, + "allocated_bytes_peak_delta": 54209017856, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:39:34.005337Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8437941248, + "inactive_split_bytes_current_delta": 8305122304, + "inactive_split_bytes_peak": 9494840320, + "inactive_split_bytes_peak_delta": 9334454272, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 95693045760, + "reserved_bytes_current_delta": 56449040384, + "reserved_bytes_peak": 95693045760, + "reserved_bytes_peak_delta": 56449040384, + "segment_count_current": 467, + "segment_count_current_delta": 396, + "segment_count_peak": 467, + "segment_count_peak_delta": 396, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T124043404331Z.pickle", + "timestamp_utc": "2026-08-01T12:40:43.519057Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T124043404331Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T124043404331Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..834c4fe5627ff971a7b733793c1af8b7f5f29f8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T124043404331Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aea2a0d0404e4e11e0fd7bfe42b13eb7051cd5d3e96753d38255910c1423d2a3 +size 5411507 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T124044834015Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T124044834015Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5e6066f73ff0e82292b59c61375056c088f6e3e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T124044834015Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113303071744, + "active_bytes_peak_delta": 23627717632, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113303071744, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113303071744, + "device": 1, + "inactive_split_bytes_current": 10501538816, + "inactive_split_bytes_peak": 10968354816, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:44.833595Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113303071744, + "allocated_bytes_peak_delta": 23627717632, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89675354112, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89675354112, + "device": 1, + "inactive_split_bytes_current": 8437941248, + "inactive_split_bytes_peak": 9494840320, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 95693045760, + "reserved_bytes_peak": 95693045760, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.538520Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10501538816, + "inactive_split_bytes_current_delta": 2063597568, + "inactive_split_bytes_peak": 10968354816, + "inactive_split_bytes_peak_delta": 1473514496, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116121403392, + "reserved_bytes_current_delta": 20428357632, + "reserved_bytes_peak": 116121403392, + "reserved_bytes_peak_delta": 20428357632, + "segment_count_current": 536, + "segment_count_current_delta": 69, + "segment_count_peak": 536, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T124044834015Z.pickle", + "timestamp_utc": "2026-08-01T12:40:44.939782Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T124044834015Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T124044834015Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..51c9fcccc309d0b43acdd4253a667e9e0f097d4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T124044834015Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b575d30fd58ccd0b5f81b04abf744b3023c649806fa48193b18898213b9dea59 +size 5567801 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T124045966999Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T124045966999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e1310c0207ed8b8821aa94268c24d122ddfdcd4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T124045966999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113312528896, + "active_bytes_peak_delta": 9457152, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312528896, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312528896, + "device": 1, + "inactive_split_bytes_current": 9562014720, + "inactive_split_bytes_peak": 10968354816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:45.966579Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113312528896, + "allocated_bytes_peak_delta": 9457152, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113303071744, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113303071744, + "device": 1, + "inactive_split_bytes_current": 10501538816, + "inactive_split_bytes_peak": 10968354816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:44.953540Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9562014720, + "inactive_split_bytes_current_delta": -939524096, + "inactive_split_bytes_peak": 10968354816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116121403392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116121403392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T124045966999Z.pickle", + "timestamp_utc": "2026-08-01T12:40:46.066943Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T124045966999Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T124045966999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..12810c612c051c195b8e2d507a6a1dedc568ba01 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T124045966999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052f8089cdce654b7ca84608bb0b638b27566e9e1c0619fc81f20439c09c75c3 +size 5563558 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T124048704726Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T124048704726Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bc72f1f9d7b9f245a45d1fb9c16bd9f40f7976e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T124048704726Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 18160741888, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17570695168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:48.704149Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 6983168, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312528896, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312528896, + "device": 1, + "inactive_split_bytes_current": 9562014720, + "inactive_split_bytes_peak": 10968354816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:46.079707Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_current_delta": 2753294336, + "inactive_split_bytes_peak": 17570695168, + "inactive_split_bytes_peak_delta": 6602340352, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149466120192, + "reserved_bytes_current_delta": 33344716800, + "reserved_bytes_peak": 149466120192, + "reserved_bytes_peak_delta": 33344716800, + "segment_count_current": 758, + "segment_count_current_delta": 222, + "segment_count_peak": 758, + "segment_count_peak_delta": 222, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T124048704726Z.pickle", + "timestamp_utc": "2026-08-01T12:40:48.821423Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T124048704726Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T124048704726Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5da8fcd3bd15933d6fc078a175193e7ef9cbbf2b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T124048704726Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7043fcf449ab66c4c8f817e8e79dbfa3562aa0caec2f4c632ce4278b492b9678 +size 5874785 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T124049788513Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T124049788513Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1368e579bfb75b96883f7f397f04ebd866dae208 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T124049788513Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_peak": 17570695168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:49.788033Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17570695168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:48.836248Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 17570695168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149466120192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149466120192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T124049788513Z.pickle", + "timestamp_utc": "2026-08-01T12:40:49.795165Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T124049788513Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T124049788513Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7b22c9a02e74e067fa262dab22da9c341af3a515 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T124049788513Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11e9cb7eb26f047dda75f2ef8a82ea21dcb8a6b3536ac361f72e1813a4c8190d +size 222691 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T124050737661Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T124050737661Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c6410b3d67030626b4cb896eba5c52c2b04521b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T124050737661Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_peak": 17570695168, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:50.737253Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_peak": 17570695168, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:49.796138Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17570695168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149466120192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149466120192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T124050737661Z.pickle", + "timestamp_utc": "2026-08-01T12:40:50.744097Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T124050737661Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T124050737661Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dea3595d985708d2c17da4eea2aa9e93a32a5920 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T124050737661Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96b11c453a727586d5b7a9177a9abb01ea32311c08b8da51cc24cc0d399469c7 +size 222691 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T124051686770Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T124051686770Z.json new file mode 100644 index 0000000000000000000000000000000000000000..886da2ce4b9b9801142fc418766a4f3db097e52d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T124051686770Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_peak": 17570695168, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:51.686361Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_peak": 17570695168, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:50.745079Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17570695168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149466120192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149466120192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T124051686770Z.pickle", + "timestamp_utc": "2026-08-01T12:40:51.693128Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T124051686770Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T124051686770Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..357661e62ab6859b9052f36997939ec9d2e75197 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T124051686770Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0b4c992588df8805c816d4302b8af793dfbd6ed2ab973a1aa364f1d34cf189a +size 222691 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T124052629454Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T124052629454Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb2bea5593ec770a806dd57407ba05abb8c562c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T124052629454Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_peak": 17570695168, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:52.629031Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_peak": 17570695168, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:51.694189Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17570695168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149466120192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149466120192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T124052629454Z.pickle", + "timestamp_utc": "2026-08-01T12:40:52.635479Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T124052629454Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T124052629454Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ee7678d32d78d5acf94301125981fb3c2cb4c21 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T124052629454Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbfbf5a788eac3a8506711424fbcccaa772282052ea51eeda35e0dc38d9177f9 +size 222691 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T124053569190Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T124053569190Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a82e95b4f8286f67b160b0c3ca4f2bc588610865 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T124053569190Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_peak": 17570695168, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:53.568750Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_peak": 17570695168, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:52.636443Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17570695168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149466120192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149466120192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T124053569190Z.pickle", + "timestamp_utc": "2026-08-01T12:40:53.575656Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T124053569190Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T124053569190Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ce452c1e6f51a7e3afae67af0785e5e287a4df6d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T124053569190Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f9e74097d8750191b7de3e1481ed1578d94b732424dcd16b45ebaf242b40454 +size 222691 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T124153083491Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T124153083491Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e0e834f5658a27b4b0cfd56c85e846199cdcbe2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T124153083491Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 6996782080, + "inactive_split_bytes_peak": 17570695168, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 150359506944, + "reserved_bytes_peak": 150359506944, + "segment_count_current": 764, + "segment_count_peak": 764, + "timestamp_utc": "2026-08-01T12:41:53.082985Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 1, + "inactive_split_bytes_current": 12313211904, + "inactive_split_bytes_peak": 17570695168, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:53.576616Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6996782080, + "inactive_split_bytes_current_delta": -5316429824, + "inactive_split_bytes_peak": 17570695168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 150359506944, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150359506944, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 764, + "segment_count_current_delta": 6, + "segment_count_peak": 764, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T124153083491Z.pickle", + "timestamp_utc": "2026-08-01T12:41:53.166132Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T124153083491Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T124153083491Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ef90c7bd1626a43e11cfee7608bfac07664ad4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T124153083491Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beb8dc6ff5e61cb3b0cc2bbb4dad0900c200c676079e203590bbd0a991573fc9 +size 3896143 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T124043404465Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T124043404465Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5b9247721e093e480e45f6abb3faf79cd1e42921 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T124043404465Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 89675354112, + "active_bytes_peak_delta": 54209017856, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89675354112, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89675354112, + "device": 2, + "inactive_split_bytes_current": 8440038400, + "inactive_split_bytes_peak": 9499034624, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 95693045760, + "reserved_bytes_peak": 95693045760, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.403976Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 89675354112, + "allocated_bytes_peak_delta": 54209017856, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:39:34.006390Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8440038400, + "inactive_split_bytes_current_delta": 8307219456, + "inactive_split_bytes_peak": 9499034624, + "inactive_split_bytes_peak_delta": 9338648576, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 95693045760, + "reserved_bytes_current_delta": 56449040384, + "reserved_bytes_peak": 95693045760, + "reserved_bytes_peak_delta": 56449040384, + "segment_count_current": 467, + "segment_count_current_delta": 396, + "segment_count_peak": 467, + "segment_count_peak_delta": 396, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T124043404465Z.pickle", + "timestamp_utc": "2026-08-01T12:40:43.517832Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T124043404465Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T124043404465Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8e0c9979c603661ae352604e45a5dcde2f445532 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T124043404465Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2dfe782f9e5e97072947ea489804af57867f1e6e1dfbcd443126448b5f2f083 +size 5411750 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T124044834030Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T124044834030Z.json new file mode 100644 index 0000000000000000000000000000000000000000..88d205de70b4f6c624e56d6841604e7ffbe0db15 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T124044834030Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113302154240, + "active_bytes_peak_delta": 23626800128, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113302154240, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113302154240, + "device": 2, + "inactive_split_bytes_current": 10679796736, + "inactive_split_bytes_peak": 10679796736, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116119306240, + "reserved_bytes_peak": 116119306240, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:40:44.833616Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113302154240, + "allocated_bytes_peak_delta": 23626800128, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89675354112, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89675354112, + "device": 2, + "inactive_split_bytes_current": 8440038400, + "inactive_split_bytes_peak": 9499034624, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 95693045760, + "reserved_bytes_peak": 95693045760, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.536145Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10679796736, + "inactive_split_bytes_current_delta": 2239758336, + "inactive_split_bytes_peak": 10679796736, + "inactive_split_bytes_peak_delta": 1180762112, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116119306240, + "reserved_bytes_current_delta": 20426260480, + "reserved_bytes_peak": 116119306240, + "reserved_bytes_peak_delta": 20426260480, + "segment_count_current": 535, + "segment_count_current_delta": 68, + "segment_count_peak": 535, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T124044834030Z.pickle", + "timestamp_utc": "2026-08-01T12:40:44.943085Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T124044834030Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T124044834030Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1352370119af38bab093178dd777cd25d5910f4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T124044834030Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:437cc88e74b8795f38307d845107c49f70c8d11fdb85a6be2b8eb585baa70c99 +size 5564172 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T124045967001Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T124045967001Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dabbbcb57cee7faea0440f15c08de92a707af69b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T124045967001Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113313537024, + "active_bytes_peak_delta": 11382784, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113313537024, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113313537024, + "device": 2, + "inactive_split_bytes_current": 9731884032, + "inactive_split_bytes_peak": 10679796736, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:45.966592Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113313537024, + "allocated_bytes_peak_delta": 11382784, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113302154240, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113302154240, + "device": 2, + "inactive_split_bytes_current": 10679796736, + "inactive_split_bytes_peak": 10679796736, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116119306240, + "reserved_bytes_peak": 116119306240, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:40:44.958561Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9731884032, + "inactive_split_bytes_current_delta": -947912704, + "inactive_split_bytes_peak": 10679796736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116121403392, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 116121403392, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 536, + "segment_count_current_delta": 1, + "segment_count_peak": 536, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T124045967001Z.pickle", + "timestamp_utc": "2026-08-01T12:40:46.074882Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T124045967001Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T124045967001Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d5c64a41af4db0611c55f0ce17e13c1817119c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T124045967001Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ceffb19a40835e70ef1c5cae6dbe62ccaf889477591bfd166d9d20b3c53cf84 +size 5556864 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T124048704626Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T124048704626Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dccc36b7ca141bb6947f2052d893c00b03ebec50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T124048704626Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131474057216, + "active_bytes_peak_delta": 18160520192, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 12543898624, + "inactive_split_bytes_peak": 17637957120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149812150272, + "reserved_bytes_peak": 149812150272, + "segment_count_current": 754, + "segment_count_peak": 754, + "timestamp_utc": "2026-08-01T12:40:48.704085Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113320298496, + "allocated_bytes_peak_delta": 6761472, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113313537024, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113313537024, + "device": 2, + "inactive_split_bytes_current": 9731884032, + "inactive_split_bytes_peak": 10679796736, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:46.089611Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12543898624, + "inactive_split_bytes_current_delta": 2812014592, + "inactive_split_bytes_peak": 17637957120, + "inactive_split_bytes_peak_delta": 6958160384, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149812150272, + "reserved_bytes_current_delta": 33690746880, + "reserved_bytes_peak": 149812150272, + "reserved_bytes_peak_delta": 33690746880, + "segment_count_current": 754, + "segment_count_current_delta": 218, + "segment_count_peak": 754, + "segment_count_peak_delta": 218, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T124048704626Z.pickle", + "timestamp_utc": "2026-08-01T12:40:48.826536Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T124048704626Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T124048704626Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a6af9155d028fa2f572578c4282a5b947d61a71f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T124048704626Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4272293f361ecd54ce7db828f0b60b7a22e31c0e8ea27d8099a645a4ef7f3f15 +size 5868753 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T124049788545Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T124049788545Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06746a58460e08a3208c82b96355e94245653f56 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T124049788545Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131474057216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 12543898624, + "inactive_split_bytes_peak": 17637957120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149812150272, + "reserved_bytes_peak": 149812150272, + "segment_count_current": 754, + "segment_count_peak": 754, + "timestamp_utc": "2026-08-01T12:40:49.788067Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113320298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 12543898624, + "inactive_split_bytes_peak": 17637957120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 149812150272, + "reserved_bytes_peak": 149812150272, + "segment_count_current": 754, + "segment_count_peak": 754, + "timestamp_utc": "2026-08-01T12:40:48.845368Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12543898624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17637957120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149812150272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149812150272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 754, + "segment_count_current_delta": 0, + "segment_count_peak": 754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T124049788545Z.pickle", + "timestamp_utc": "2026-08-01T12:40:49.795650Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T124049788545Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T124049788545Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fbeb6a38400fe9323bc38c65113d2c7e20205fce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T124049788545Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3886d194cbc0ae8b76541700508a6b784095b7d2fc1ad2c31f6e1ec6ec6f1e3f +size 220817 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T124050737678Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T124050737678Z.json new file mode 100644 index 0000000000000000000000000000000000000000..89fe9d447e5b0671adb09c64dc057d968e2b0aee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T124050737678Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131474057216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 12541801472, + "inactive_split_bytes_peak": 17637957120, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149812150272, + "reserved_bytes_peak": 149812150272, + "segment_count_current": 754, + "segment_count_peak": 754, + "timestamp_utc": "2026-08-01T12:40:50.737262Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113320298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 12543898624, + "inactive_split_bytes_peak": 17637957120, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 149812150272, + "reserved_bytes_peak": 149812150272, + "segment_count_current": 754, + "segment_count_peak": 754, + "timestamp_utc": "2026-08-01T12:40:49.796629Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12541801472, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 17637957120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149812150272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149812150272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 754, + "segment_count_current_delta": 0, + "segment_count_peak": 754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T124050737678Z.pickle", + "timestamp_utc": "2026-08-01T12:40:50.744084Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T124050737678Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T124050737678Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c76003ed6e08b3ae6e6bdd941974f993eea712f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T124050737678Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3409d01b568aa3f9f9ca45f30c641dfda125adb552a3e8a29f2928ed44baec79 +size 220814 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T124051686797Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T124051686797Z.json new file mode 100644 index 0000000000000000000000000000000000000000..35b7e22fb063e93dd40730c33e1c9904916c11bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T124051686797Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131474057216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 12541801472, + "inactive_split_bytes_peak": 17637957120, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149812150272, + "reserved_bytes_peak": 149812150272, + "segment_count_current": 754, + "segment_count_peak": 754, + "timestamp_utc": "2026-08-01T12:40:51.686380Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113320298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 12541801472, + "inactive_split_bytes_peak": 17637957120, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 149812150272, + "reserved_bytes_peak": 149812150272, + "segment_count_current": 754, + "segment_count_peak": 754, + "timestamp_utc": "2026-08-01T12:40:50.745022Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12541801472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17637957120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149812150272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149812150272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 754, + "segment_count_current_delta": 0, + "segment_count_peak": 754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T124051686797Z.pickle", + "timestamp_utc": "2026-08-01T12:40:51.693212Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T124051686797Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T124051686797Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..141e7d995ccd8788a512d697c424f50538701ddd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T124051686797Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e19873a4ab6e8c1469b6a36224e2117efe3b3fdb7ef33af4425827d034fb692 +size 220845 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T124052629462Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T124052629462Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a40ead81ea1d47fb9223c21d64fb31b0eca3e05f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T124052629462Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131474057216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 12541801472, + "inactive_split_bytes_peak": 17637957120, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149812150272, + "reserved_bytes_peak": 149812150272, + "segment_count_current": 754, + "segment_count_peak": 754, + "timestamp_utc": "2026-08-01T12:40:52.629059Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113320298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 12541801472, + "inactive_split_bytes_peak": 17637957120, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 149812150272, + "reserved_bytes_peak": 149812150272, + "segment_count_current": 754, + "segment_count_peak": 754, + "timestamp_utc": "2026-08-01T12:40:51.694222Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12541801472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17637957120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149812150272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149812150272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 754, + "segment_count_current_delta": 0, + "segment_count_peak": 754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T124052629462Z.pickle", + "timestamp_utc": "2026-08-01T12:40:52.635396Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T124052629462Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T124052629462Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d0a637fbf244e598481722b52a01fc74c34c8b18 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T124052629462Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b48e651f9f2dc3861a2ef787c00254e9fbc5a435176a99cbdc3db46fe65bbe0 +size 220814 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T124053569304Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T124053569304Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0755550dd1022ef0e687ca84f2d9775dd66f994b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T124053569304Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131474057216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 12541801472, + "inactive_split_bytes_peak": 17637957120, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149812150272, + "reserved_bytes_peak": 149812150272, + "segment_count_current": 754, + "segment_count_peak": 754, + "timestamp_utc": "2026-08-01T12:40:53.568845Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113320298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 12541801472, + "inactive_split_bytes_peak": 17637957120, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 149812150272, + "reserved_bytes_peak": 149812150272, + "segment_count_current": 754, + "segment_count_peak": 754, + "timestamp_utc": "2026-08-01T12:40:52.636297Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12541801472, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17637957120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149812150272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149812150272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 754, + "segment_count_current_delta": 0, + "segment_count_peak": 754, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T124053569304Z.pickle", + "timestamp_utc": "2026-08-01T12:40:53.576183Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T124053569304Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T124053569304Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..19ed6f26f023eb2323199aa78e350093e4e17e4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T124053569304Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb2f4670228e5733f49e2556218ef1f1d54e34c6cb07df6c7177f48f4a5a40bf +size 220845 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T124153083597Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T124153083597Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7eb632e93f4bc7f5b5799771c3ecc16b73f77d00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T124153083597Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 131474057216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 6879341568, + "inactive_split_bytes_peak": 17637957120, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149881356288, + "reserved_bytes_peak": 149881356288, + "segment_count_current": 759, + "segment_count_peak": 759, + "timestamp_utc": "2026-08-01T12:41:53.083065Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 113320298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131474057216, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113320298496, + "device": 2, + "inactive_split_bytes_current": 12541801472, + "inactive_split_bytes_peak": 17637957120, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 149812150272, + "reserved_bytes_peak": 149812150272, + "segment_count_current": 754, + "segment_count_peak": 754, + "timestamp_utc": "2026-08-01T12:40:53.577123Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6879341568, + "inactive_split_bytes_current_delta": -5662459904, + "inactive_split_bytes_peak": 17637957120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149881356288, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 149881356288, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 759, + "segment_count_current_delta": 5, + "segment_count_peak": 759, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T124153083597Z.pickle", + "timestamp_utc": "2026-08-01T12:41:53.177200Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T124153083597Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T124153083597Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d66d7733b0910567078f15afe1ab2595a6d8a61c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T124153083597Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:285028bac1bf198857c3d88b5f09c67c112e648afae526b522f6ec2158fd5ce4 +size 3970712 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T124043404394Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T124043404394Z.json new file mode 100644 index 0000000000000000000000000000000000000000..73600a805190a6ff0eb3440fcbbb732abe1917b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T124043404394Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 89675354112, + "active_bytes_peak_delta": 54209017856, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89675354112, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89675354112, + "device": 3, + "inactive_split_bytes_current": 8437941248, + "inactive_split_bytes_peak": 9494840320, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 95693045760, + "reserved_bytes_peak": 95693045760, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.403901Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 89675354112, + "allocated_bytes_peak_delta": 54209017856, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:39:34.005370Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8437941248, + "inactive_split_bytes_current_delta": 8305122304, + "inactive_split_bytes_peak": 9494840320, + "inactive_split_bytes_peak_delta": 9334454272, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 95693045760, + "reserved_bytes_current_delta": 56449040384, + "reserved_bytes_peak": 95693045760, + "reserved_bytes_peak_delta": 56449040384, + "segment_count_current": 467, + "segment_count_current_delta": 396, + "segment_count_peak": 467, + "segment_count_peak_delta": 396, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T124043404394Z.pickle", + "timestamp_utc": "2026-08-01T12:40:43.514915Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T124043404394Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T124043404394Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ae36af2b7d14c0ee6f07f08f4884302b05e770d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T124043404394Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8632b8d3175e273afdbd6195294382c719af09382cdf6d846a10680dbf8898cf +size 5411480 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T124044834034Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T124044834034Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ea3cdd75c9e9f9bfed5671cd5444c5364721584 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T124044834034Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113303071744, + "active_bytes_peak_delta": 23627717632, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113303071744, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113303071744, + "device": 3, + "inactive_split_bytes_current": 10501538816, + "inactive_split_bytes_peak": 10968354816, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:44.833623Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113303071744, + "allocated_bytes_peak_delta": 23627717632, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89675354112, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89675354112, + "device": 3, + "inactive_split_bytes_current": 8437941248, + "inactive_split_bytes_peak": 9494840320, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 95693045760, + "reserved_bytes_peak": 95693045760, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.530092Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10501538816, + "inactive_split_bytes_current_delta": 2063597568, + "inactive_split_bytes_peak": 10968354816, + "inactive_split_bytes_peak_delta": 1473514496, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116121403392, + "reserved_bytes_current_delta": 20428357632, + "reserved_bytes_peak": 116121403392, + "reserved_bytes_peak_delta": 20428357632, + "segment_count_current": 536, + "segment_count_current_delta": 69, + "segment_count_peak": 536, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T124044834034Z.pickle", + "timestamp_utc": "2026-08-01T12:40:44.940241Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T124044834034Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T124044834034Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aece56dacd6ca5392737a2acf11ad7f2d3e131dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T124044834034Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddc59dd1efb68cdabda13bbf29cb642875442724db08c3a60999422b5dcec22c +size 5561538 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T124045967022Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T124045967022Z.json new file mode 100644 index 0000000000000000000000000000000000000000..962a5a2f8b707718a58731a8d9aedf771f2b774d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T124045967022Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113312528896, + "active_bytes_peak_delta": 9457152, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312528896, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312528896, + "device": 3, + "inactive_split_bytes_current": 9562014720, + "inactive_split_bytes_peak": 10968354816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:45.966621Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113312528896, + "allocated_bytes_peak_delta": 9457152, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113303071744, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113303071744, + "device": 3, + "inactive_split_bytes_current": 10501538816, + "inactive_split_bytes_peak": 10968354816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:44.953739Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9562014720, + "inactive_split_bytes_current_delta": -939524096, + "inactive_split_bytes_peak": 10968354816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116121403392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116121403392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T124045967022Z.pickle", + "timestamp_utc": "2026-08-01T12:40:46.066228Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T124045967022Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T124045967022Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ee46754768698ff38770a2a7ba5207e3c4d54c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T124045967022Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e02afab99373a2d0aa66ba0173a4c2ad414c5491fe0fb3e28e3a292a9627e4 +size 5563315 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T124048705766Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T124048705766Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8bd61312b09700eea5194ffaf7a43b67673db208 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T124048705766Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 18160741888, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17574889472, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:48.704831Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 6983168, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312528896, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312528896, + "device": 3, + "inactive_split_bytes_current": 9562014720, + "inactive_split_bytes_peak": 10968354816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:46.079149Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_current_delta": 2753294336, + "inactive_split_bytes_peak": 17574889472, + "inactive_split_bytes_peak_delta": 6606534656, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149466120192, + "reserved_bytes_current_delta": 33344716800, + "reserved_bytes_peak": 149466120192, + "reserved_bytes_peak_delta": 33344716800, + "segment_count_current": 758, + "segment_count_current_delta": 222, + "segment_count_peak": 758, + "segment_count_peak_delta": 222, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T124048705766Z.pickle", + "timestamp_utc": "2026-08-01T12:40:48.831088Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T124048705766Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T124048705766Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..729c759c12a109414020b3bce7021fd888dbb9d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T124048705766Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dfbf07272c5c3fe844a00a410d06c2266dbf4f26fdd6c089ca9de0b771fc1fa +size 5874868 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T124049788687Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T124049788687Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ceab69455128c0d128476a8c818ecae0b5231b53 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T124049788687Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17574889472, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:49.788151Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17574889472, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:48.848133Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17574889472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149466120192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149466120192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T124049788687Z.pickle", + "timestamp_utc": "2026-08-01T12:40:49.795692Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T124049788687Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T124049788687Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..55bc2421a164bdd300af0d9604c4f8c996b86856 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T124049788687Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87f416209d763ef846cf4827cc6e698924fc4d2c2f72aeae221ed1d60197012d +size 222746 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T124050737795Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T124050737795Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eda55e8c7d3dad4c1252595a16d39a9736c03e4c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T124050737795Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17574889472, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:50.737368Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17574889472, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:49.796690Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17574889472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149466120192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149466120192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T124050737795Z.pickle", + "timestamp_utc": "2026-08-01T12:40:50.744685Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T124050737795Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T124050737795Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c03fbeafba54e557d8c890ac390926766a9f936a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T124050737795Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:916d46981d2d6bf6b85c95029b583d30cec25880dc208f19b3ba1f31bda333f8 +size 222746 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T124051687439Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T124051687439Z.json new file mode 100644 index 0000000000000000000000000000000000000000..985ee4159b5bb1789866977792a2591e3561c8fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T124051687439Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17574889472, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:51.686883Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17574889472, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:50.745645Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17574889472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149466120192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149466120192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T124051687439Z.pickle", + "timestamp_utc": "2026-08-01T12:40:51.696151Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T124051687439Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T124051687439Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e83884475aadb3481f2f102d1b1660ab8d7f7503 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T124051687439Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96c841776d6c9830db30aa1d07fa3bcb93ae7069d8976354a04b1da7f06f8b8e +size 222746 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T124052629571Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T124052629571Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0361b36f9e777a4e0f5f3ed19ae8997e6591de28 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T124052629571Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17574889472, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:52.629159Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17574889472, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:51.697387Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17574889472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149466120192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149466120192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T124052629571Z.pickle", + "timestamp_utc": "2026-08-01T12:40:52.635944Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T124052629571Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T124052629571Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5c91f4eea2841f8bfdb796a65b68f0f4cabc2d68 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T124052629571Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc6f9449b4449e41539c58828b43d2d1e60bf2ac54c600d410d3fc392d43ebbf +size 222746 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T124053569309Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T124053569309Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f38c5f4418a32d72e71a5621dbd09a5a43a9da88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T124053569309Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17574889472, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:53.568873Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17574889472, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:52.636909Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17574889472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149466120192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149466120192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T124053569309Z.pickle", + "timestamp_utc": "2026-08-01T12:40:53.576033Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T124053569309Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T124053569309Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5cca70fbf7f686260e0063c6ce9ef09bd53e719c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T124053569309Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e99217897a1cea0a3c5be93163bdfdde6a9d8599ab42a819092c9ba81b7df6c6 +size 222746 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T124153083527Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T124153083527Z.json new file mode 100644 index 0000000000000000000000000000000000000000..823976c0f8a6e3f0866ba067cfe779a0cc15ad3d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T124153083527Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 131473270784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 6998879232, + "inactive_split_bytes_peak": 17574889472, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 150359506944, + "reserved_bytes_peak": 150359506944, + "segment_count_current": 764, + "segment_count_peak": 764, + "timestamp_utc": "2026-08-01T12:41:53.083025Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 113319512064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131473270784, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319512064, + "device": 3, + "inactive_split_bytes_current": 12315309056, + "inactive_split_bytes_peak": 17574889472, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149466120192, + "reserved_bytes_peak": 149466120192, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:53.577049Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6998879232, + "inactive_split_bytes_current_delta": -5316429824, + "inactive_split_bytes_peak": 17574889472, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 150359506944, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150359506944, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 764, + "segment_count_current_delta": 6, + "segment_count_peak": 764, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T124153083527Z.pickle", + "timestamp_utc": "2026-08-01T12:41:53.178725Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T124153083527Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T124153083527Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ace5e45b7e174d4c3f7ebe84860f63dca5b15c43 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T124153083527Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2bbe2e935eb388b88ca92864edeba12e43b7665e2b9e23223189efbb4af61d9 +size 3896169 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T124043404364Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T124043404364Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e68946b6b8c62449e5d16e830d314eecce2f2b56 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T124043404364Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66923303936, + "active_bytes_current_delta": 31691406336, + "active_bytes_peak": 89672994304, + "active_bytes_peak_delta": 54206215680, + "after": { + "active_bytes_current": 66923303936, + "active_bytes_peak": 89672994304, + "allocated_bytes_current": 66923303936, + "allocated_bytes_peak": 89672994304, + "device": 4, + "inactive_split_bytes_current": 9962482688, + "inactive_split_bytes_peak": 11078168576, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 95720308736, + "reserved_bytes_peak": 95720308736, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:40:43.403891Z", + "world_size": 8 + }, + "allocated_bytes_current": 66923303936, + "allocated_bytes_current_delta": 31691406336, + "allocated_bytes_peak": 89672994304, + "allocated_bytes_peak_delta": 54206215680, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:39:34.005552Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9962482688, + "inactive_split_bytes_current_delta": 9830106112, + "inactive_split_bytes_peak": 11078168576, + "inactive_split_bytes_peak_delta": 10924759040, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 95720308736, + "reserved_bytes_current_delta": 56476303360, + "reserved_bytes_peak": 95720308736, + "reserved_bytes_peak_delta": 56476303360, + "segment_count_current": 468, + "segment_count_current_delta": 397, + "segment_count_peak": 468, + "segment_count_peak_delta": 397, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T124043404364Z.pickle", + "timestamp_utc": "2026-08-01T12:40:43.517846Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T124043404364Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T124043404364Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4cfc20b10b18511039dc6f9aeea898e5523e3357 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T124043404364Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03a925df18ec11821f4dbf2fc47e0889c65edda31d67f83baa67b7123028905f +size 5467919 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T124044833993Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T124044833993Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5c7be5ce13d80682d2bfa4ec72765e8d847fd45 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T124044833993Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66923303936, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113305017344, + "active_bytes_peak_delta": 23632023040, + "after": { + "active_bytes_current": 66923303936, + "active_bytes_peak": 113305017344, + "allocated_bytes_current": 66923303936, + "allocated_bytes_peak": 113305017344, + "device": 4, + "inactive_split_bytes_current": 11061390336, + "inactive_split_bytes_peak": 12075125760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115620184064, + "reserved_bytes_peak": 115620184064, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T12:40:44.833571Z", + "world_size": 8 + }, + "allocated_bytes_current": 66923303936, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113305017344, + "allocated_bytes_peak_delta": 23632023040, + "before": { + "active_bytes_current": 66923303936, + "active_bytes_peak": 89672994304, + "allocated_bytes_current": 66923303936, + "allocated_bytes_peak": 89672994304, + "device": 4, + "inactive_split_bytes_current": 9962482688, + "inactive_split_bytes_peak": 11078168576, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 95720308736, + "reserved_bytes_peak": 95720308736, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:40:43.536527Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11061390336, + "inactive_split_bytes_current_delta": 1098907648, + "inactive_split_bytes_peak": 12075125760, + "inactive_split_bytes_peak_delta": 996957184, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115620184064, + "reserved_bytes_current_delta": 19899875328, + "reserved_bytes_peak": 115620184064, + "reserved_bytes_peak_delta": 19899875328, + "segment_count_current": 531, + "segment_count_current_delta": 63, + "segment_count_peak": 531, + "segment_count_peak_delta": 63, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T124044833993Z.pickle", + "timestamp_utc": "2026-08-01T12:40:44.941293Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T124044833993Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T124044833993Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..688acf24c791bf6797b0ca8c34ce45f531bb9d13 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T124044833993Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcbf6ec57c1ec915f9ca7ffe76f66beab560cf06f1ed2d91211c6320844430b0 +size 5570296 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T124045967022Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T124045967022Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2b399074a85238e8517233ad0551ad1f6923b5ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T124045967022Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66923303936, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113307900416, + "active_bytes_peak_delta": 2883072, + "after": { + "active_bytes_current": 66923303936, + "active_bytes_peak": 113307900416, + "allocated_bytes_current": 66923303936, + "allocated_bytes_peak": 113307900416, + "device": 4, + "inactive_split_bytes_current": 10981698560, + "inactive_split_bytes_peak": 12075125760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115620184064, + "reserved_bytes_peak": 115620184064, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T12:40:45.966604Z", + "world_size": 8 + }, + "allocated_bytes_current": 66923303936, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113307900416, + "allocated_bytes_peak_delta": 2883072, + "before": { + "active_bytes_current": 66923303936, + "active_bytes_peak": 113305017344, + "allocated_bytes_current": 66923303936, + "allocated_bytes_peak": 113305017344, + "device": 4, + "inactive_split_bytes_current": 11061390336, + "inactive_split_bytes_peak": 12075125760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115620184064, + "reserved_bytes_peak": 115620184064, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T12:40:44.956711Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10981698560, + "inactive_split_bytes_current_delta": -79691776, + "inactive_split_bytes_peak": 12075125760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115620184064, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115620184064, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 531, + "segment_count_current_delta": 0, + "segment_count_peak": 531, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T124045967022Z.pickle", + "timestamp_utc": "2026-08-01T12:40:46.076651Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T124045967022Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T124045967022Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a747650a158b3df1d06108f720b69a24941f29ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T124045967022Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3f31569866434978b1837946ebead00d9b7a355180997654ce5606f14984f3f +size 5580473 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T124048704783Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T124048704783Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e20d122ea7821b907ce2ff07ad5c6ad6ac3b888 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T124048704783Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66940347392, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131480305664, + "active_bytes_peak_delta": 18172405248, + "after": { + "active_bytes_current": 66940347392, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66940347392, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_peak": 19648360448, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 150921543680, + "reserved_bytes_peak": 150921543680, + "segment_count_current": 768, + "segment_count_peak": 768, + "timestamp_utc": "2026-08-01T12:40:48.704169Z", + "world_size": 8 + }, + "allocated_bytes_current": 66940347392, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113326546944, + "allocated_bytes_peak_delta": 18646528, + "before": { + "active_bytes_current": 66923303936, + "active_bytes_peak": 113307900416, + "allocated_bytes_current": 66923303936, + "allocated_bytes_peak": 113307900416, + "device": 4, + "inactive_split_bytes_current": 10981698560, + "inactive_split_bytes_peak": 12075125760, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115620184064, + "reserved_bytes_peak": 115620184064, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T12:40:46.093987Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_current_delta": 3598446592, + "inactive_split_bytes_peak": 19648360448, + "inactive_split_bytes_peak_delta": 7573234688, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 150921543680, + "reserved_bytes_current_delta": 35301359616, + "reserved_bytes_peak": 150921543680, + "reserved_bytes_peak_delta": 35301359616, + "segment_count_current": 768, + "segment_count_current_delta": 237, + "segment_count_peak": 768, + "segment_count_peak_delta": 237, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T124048704783Z.pickle", + "timestamp_utc": "2026-08-01T12:40:48.824627Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T124048704783Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T124048704783Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a305cc42e6516dd422071d92879f96a913d8bba6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T124048704783Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc3190df6627d2d0bd450b6e29ca534719ca17bbe0c4c3ef52ff06ca3adf3b89 +size 5879838 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T124049788580Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T124049788580Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fad3e9dd876a81f6f1b0b4a21415e86d92c17006 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T124049788580Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66940347392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131480305664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66940347392, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66940347392, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_peak": 19648360448, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 150921543680, + "reserved_bytes_peak": 150921543680, + "segment_count_current": 768, + "segment_count_peak": 768, + "timestamp_utc": "2026-08-01T12:40:49.788076Z", + "world_size": 8 + }, + "allocated_bytes_current": 66940347392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113326546944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66940347392, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66940347392, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_peak": 19648360448, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 150921543680, + "reserved_bytes_peak": 150921543680, + "segment_count_current": 768, + "segment_count_peak": 768, + "timestamp_utc": "2026-08-01T12:40:48.841114Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19648360448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 150921543680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150921543680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 768, + "segment_count_current_delta": 0, + "segment_count_peak": 768, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T124049788580Z.pickle", + "timestamp_utc": "2026-08-01T12:40:49.796012Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T124049788580Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T124049788580Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e29f30c36b65c346312277f0a94a1443710fa872 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T124049788580Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9339d04c97ff71e5a6496ac01f9815e416c0845ba3d75f987773f2719ba670c6 +size 232115 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T124050737664Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T124050737664Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f4152ce34985ff5cfa8704027c4711167c9fbbac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T124050737664Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66940347392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131480305664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66940347392, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66940347392, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_peak": 19648360448, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 150921543680, + "reserved_bytes_peak": 150921543680, + "segment_count_current": 768, + "segment_count_peak": 768, + "timestamp_utc": "2026-08-01T12:40:50.737246Z", + "world_size": 8 + }, + "allocated_bytes_current": 66940347392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113326546944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66940347392, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66940347392, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_peak": 19648360448, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 150921543680, + "reserved_bytes_peak": 150921543680, + "segment_count_current": 768, + "segment_count_peak": 768, + "timestamp_utc": "2026-08-01T12:40:49.797072Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19648360448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 150921543680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150921543680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 768, + "segment_count_current_delta": 0, + "segment_count_peak": 768, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T124050737664Z.pickle", + "timestamp_utc": "2026-08-01T12:40:50.744323Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T124050737664Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T124050737664Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ad7c1594a73f65eedd5472172f659e9f3fe2becb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T124050737664Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:687258d01b0dacace7c83442f1f6eb69a511805651401886f1acd1c444b7e989 +size 232115 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T124051686792Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T124051686792Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5db607bc55425d4fc16ef634dfaf955078725ed8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T124051686792Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66940347392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131480305664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66940347392, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66940347392, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_peak": 19648360448, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 150921543680, + "reserved_bytes_peak": 150921543680, + "segment_count_current": 768, + "segment_count_peak": 768, + "timestamp_utc": "2026-08-01T12:40:51.686374Z", + "world_size": 8 + }, + "allocated_bytes_current": 66940347392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113326546944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66940347392, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66940347392, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_peak": 19648360448, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 150921543680, + "reserved_bytes_peak": 150921543680, + "segment_count_current": 768, + "segment_count_peak": 768, + "timestamp_utc": "2026-08-01T12:40:50.745272Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19648360448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 150921543680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150921543680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 768, + "segment_count_current_delta": 0, + "segment_count_peak": 768, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T124051686792Z.pickle", + "timestamp_utc": "2026-08-01T12:40:51.693463Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T124051686792Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T124051686792Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c8677aa7853139d17ce4366258aa89fa4175a77 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T124051686792Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cbe021f121045bd65f7261fb4778634e74614ca0c9e9364b014e771c341f957 +size 232115 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T124052629451Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T124052629451Z.json new file mode 100644 index 0000000000000000000000000000000000000000..008c32862430ce0e0cc0d82c9ef7ebbce1c2f007 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T124052629451Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66940347392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131480305664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66940347392, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66940347392, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_peak": 19648360448, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 150921543680, + "reserved_bytes_peak": 150921543680, + "segment_count_current": 768, + "segment_count_peak": 768, + "timestamp_utc": "2026-08-01T12:40:52.629038Z", + "world_size": 8 + }, + "allocated_bytes_current": 66940347392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113326546944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66940347392, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66940347392, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_peak": 19648360448, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 150921543680, + "reserved_bytes_peak": 150921543680, + "segment_count_current": 768, + "segment_count_peak": 768, + "timestamp_utc": "2026-08-01T12:40:51.694479Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19648360448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 150921543680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150921543680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 768, + "segment_count_current_delta": 0, + "segment_count_peak": 768, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T124052629451Z.pickle", + "timestamp_utc": "2026-08-01T12:40:52.635702Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T124052629451Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T124052629451Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..274e7e1b06df1cb70fda1139470dc20e34baab99 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T124052629451Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9053710a1a8b0b3f50ab97d0fd5b9af993f132ee05e84f0727ad3319d5930013 +size 232115 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T124053569211Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T124053569211Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3469f2866b93a9fb7f2b00009c70e264763f8c3d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T124053569211Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66940347392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131480305664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66940347392, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66940347392, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_peak": 19648360448, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 150921543680, + "reserved_bytes_peak": 150921543680, + "segment_count_current": 768, + "segment_count_peak": 768, + "timestamp_utc": "2026-08-01T12:40:53.568772Z", + "world_size": 8 + }, + "allocated_bytes_current": 66940347392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113326546944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66940347392, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66940347392, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_peak": 19648360448, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 150921543680, + "reserved_bytes_peak": 150921543680, + "segment_count_current": 768, + "segment_count_peak": 768, + "timestamp_utc": "2026-08-01T12:40:52.636633Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19648360448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 150921543680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150921543680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 768, + "segment_count_current_delta": 0, + "segment_count_peak": 768, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T124053569211Z.pickle", + "timestamp_utc": "2026-08-01T12:40:53.576064Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T124053569211Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T124053569211Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..12169fb41ffc851bad854b109705ab4d5d9047b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T124053569211Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd3a830ce06ef5859283dd8762e11f07ae7d71e84268234e984b72b4fe35360e +size 232115 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T124153083483Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T124153083483Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d602b66fd4f11d91384895cc5ef017428731eb32 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T124153083483Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66957274112, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 131480305664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66957274112, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66957274112, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 9381155840, + "inactive_split_bytes_peak": 19648360448, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 151814930432, + "reserved_bytes_peak": 151814930432, + "segment_count_current": 774, + "segment_count_peak": 774, + "timestamp_utc": "2026-08-01T12:41:53.082995Z", + "world_size": 8 + }, + "allocated_bytes_current": 66957274112, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 113326546944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66940347392, + "active_bytes_peak": 131480305664, + "allocated_bytes_current": 66940347392, + "allocated_bytes_peak": 113326546944, + "device": 4, + "inactive_split_bytes_current": 14580145152, + "inactive_split_bytes_peak": 19648360448, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 150921543680, + "reserved_bytes_peak": 150921543680, + "segment_count_current": 768, + "segment_count_peak": 768, + "timestamp_utc": "2026-08-01T12:40:53.577048Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9381155840, + "inactive_split_bytes_current_delta": -5198989312, + "inactive_split_bytes_peak": 19648360448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 151814930432, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 151814930432, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 774, + "segment_count_current_delta": 6, + "segment_count_peak": 774, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T124153083483Z.pickle", + "timestamp_utc": "2026-08-01T12:41:53.170148Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T124153083483Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T124153083483Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c59f3e5a2d989b2a2d19e7a73dd593c167af5087 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T124153083483Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c02eb2b403e9098babb2cbf9d6c63bbda3cf335c439e592403c34b1a3071d8f8 +size 3981062 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T124043404476Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T124043404476Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed08e1bfaf3746baee942af2620bd15d14e9398a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T124043404476Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 89675354112, + "active_bytes_peak_delta": 54209017856, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89675354112, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89675354112, + "device": 5, + "inactive_split_bytes_current": 8440038400, + "inactive_split_bytes_peak": 9499034624, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 95693045760, + "reserved_bytes_peak": 95693045760, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.404002Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 89675354112, + "allocated_bytes_peak_delta": 54209017856, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:39:34.006053Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8440038400, + "inactive_split_bytes_current_delta": 8307219456, + "inactive_split_bytes_peak": 9499034624, + "inactive_split_bytes_peak_delta": 9338648576, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 95693045760, + "reserved_bytes_current_delta": 56449040384, + "reserved_bytes_peak": 95693045760, + "reserved_bytes_peak_delta": 56449040384, + "segment_count_current": 467, + "segment_count_current_delta": 396, + "segment_count_peak": 467, + "segment_count_peak_delta": 396, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T124043404476Z.pickle", + "timestamp_utc": "2026-08-01T12:40:43.524426Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T124043404476Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T124043404476Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..af91757a8e5a5aeb27b0989a4c16c84d64eb6f6a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T124043404476Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6a916aa8c266c50f6d4ca1a714ab043f567cd4d388640cb815b2931ed596c9a +size 5411751 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T124044834014Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T124044834014Z.json new file mode 100644 index 0000000000000000000000000000000000000000..284c88dcae7f6c7835c19e273bbf9f783dcad924 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T124044834014Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113303071744, + "active_bytes_peak_delta": 23627717632, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113303071744, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113303071744, + "device": 5, + "inactive_split_bytes_current": 10505733120, + "inactive_split_bytes_peak": 10972549120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:44.833613Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113303071744, + "allocated_bytes_peak_delta": 23627717632, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89675354112, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89675354112, + "device": 5, + "inactive_split_bytes_current": 8440038400, + "inactive_split_bytes_peak": 9499034624, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 95693045760, + "reserved_bytes_peak": 95693045760, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.541277Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10505733120, + "inactive_split_bytes_current_delta": 2065694720, + "inactive_split_bytes_peak": 10972549120, + "inactive_split_bytes_peak_delta": 1473514496, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116121403392, + "reserved_bytes_current_delta": 20428357632, + "reserved_bytes_peak": 116121403392, + "reserved_bytes_peak_delta": 20428357632, + "segment_count_current": 536, + "segment_count_current_delta": 69, + "segment_count_peak": 536, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T124044834014Z.pickle", + "timestamp_utc": "2026-08-01T12:40:44.939169Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T124044834014Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T124044834014Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f61ac6e73e019c593251a84301dbd53ef799af95 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T124044834014Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5823471d23810d029c0da61ce5192f6afbaa058882ace607683c0d7ccd131bfb +size 5561636 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T124045966999Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T124045966999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..052daebe6066cfcd5e454194f8facba1f9aa08e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T124045966999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113312528896, + "active_bytes_peak_delta": 9457152, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312528896, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312528896, + "device": 5, + "inactive_split_bytes_current": 9557820416, + "inactive_split_bytes_peak": 10972549120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:45.966602Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113312528896, + "allocated_bytes_peak_delta": 9457152, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113303071744, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113303071744, + "device": 5, + "inactive_split_bytes_current": 10505733120, + "inactive_split_bytes_peak": 10972549120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:44.951998Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9557820416, + "inactive_split_bytes_current_delta": -947912704, + "inactive_split_bytes_peak": 10972549120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116121403392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116121403392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T124045966999Z.pickle", + "timestamp_utc": "2026-08-01T12:40:46.068121Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T124045966999Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T124045966999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ed582e57a6eee76af5ce26bc87d7a11370684fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T124045966999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8110b4ac1b5bf48054676ee97ee9b9aac4243482419148c435e099c505b59e52 +size 5563449 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T124048704700Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T124048704700Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8167443b13bf36b4a9f346d65e3dd7849cce6082 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T124048704700Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131469076480, + "active_bytes_peak_delta": 18156547584, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_peak": 17298087424, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149491286016, + "reserved_bytes_peak": 149491286016, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:48.704136Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113319511552, + "allocated_bytes_peak_delta": 6982656, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312528896, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312528896, + "device": 5, + "inactive_split_bytes_current": 9557820416, + "inactive_split_bytes_peak": 10972549120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116121403392, + "reserved_bytes_peak": 116121403392, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T12:40:46.079483Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_current_delta": 2015096832, + "inactive_split_bytes_peak": 17298087424, + "inactive_split_bytes_peak_delta": 6325538304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149491286016, + "reserved_bytes_current_delta": 33369882624, + "reserved_bytes_peak": 149491286016, + "reserved_bytes_peak_delta": 33369882624, + "segment_count_current": 758, + "segment_count_current_delta": 222, + "segment_count_peak": 758, + "segment_count_peak_delta": 222, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T124048704700Z.pickle", + "timestamp_utc": "2026-08-01T12:40:48.821183Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T124048704700Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T124048704700Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f4b76991ed68cb581efabfd2530dab71cb6c6833 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T124048704700Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5645cdcb97d1d89e01f80d2fa255ea54dbb208f2cb8324cd3b7d598faee8942 +size 5864960 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T124049788512Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T124049788512Z.json new file mode 100644 index 0000000000000000000000000000000000000000..131477d4bca837f7cf9f232c74c029257e778003 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T124049788512Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131469076480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_peak": 17298087424, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149491286016, + "reserved_bytes_peak": 149491286016, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:49.788041Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319511552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_peak": 17298087424, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149491286016, + "reserved_bytes_peak": 149491286016, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:48.835536Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17298087424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149491286016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149491286016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T124049788512Z.pickle", + "timestamp_utc": "2026-08-01T12:40:49.795056Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T124049788512Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T124049788512Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..632e64680fbf8f6e1b1c07c1490a0db6291ceb56 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T124049788512Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fdaf90a877c6e5cacb5de9ec15124ae8934dabc5cceda6fc401c1aa364f1d2d +size 221668 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T124050737685Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T124050737685Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a2602f5005602a5cb010665c04fd6f4dc86fe43d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T124050737685Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131469076480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_peak": 17298087424, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149491286016, + "reserved_bytes_peak": 149491286016, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:50.737276Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319511552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_peak": 17298087424, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149491286016, + "reserved_bytes_peak": 149491286016, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:49.796003Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17298087424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149491286016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149491286016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T124050737685Z.pickle", + "timestamp_utc": "2026-08-01T12:40:50.743964Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T124050737685Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T124050737685Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14028b06d810d7a417087e14fadb7e04ca80518e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T124050737685Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f3fee763f3fdde14f69b55acc68e959ca3fdc950f910977f504bae6c59c96d3 +size 221668 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T124051686810Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T124051686810Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed672d53b0f7eaffcec70cf6086116be26ac477b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T124051686810Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131469076480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_peak": 17298087424, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149491286016, + "reserved_bytes_peak": 149491286016, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:51.686398Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319511552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_peak": 17298087424, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149491286016, + "reserved_bytes_peak": 149491286016, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:50.744884Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17298087424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149491286016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149491286016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T124051686810Z.pickle", + "timestamp_utc": "2026-08-01T12:40:51.693148Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T124051686810Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T124051686810Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..465d03382f6860a7882539c4905621898350a3cf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T124051686810Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2c767543ed82397c9f2b0de53cca2c0ba781374cc7de2d4c0a4ba8657a6d740 +size 221668 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T124052629493Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T124052629493Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6a425b28fb5d8bb2c4a12db511ed9b4e1af3aace --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T124052629493Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131469076480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_peak": 17298087424, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149491286016, + "reserved_bytes_peak": 149491286016, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:52.629074Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319511552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_peak": 17298087424, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149491286016, + "reserved_bytes_peak": 149491286016, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:51.694196Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17298087424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149491286016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149491286016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T124052629493Z.pickle", + "timestamp_utc": "2026-08-01T12:40:52.635638Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T124052629493Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T124052629493Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..45e3c1ef8385d9cfb33c27952cc7d6335d9d762b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T124052629493Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b257139d63e472a3dea5e7334a06edfe87055dd7e4bc5f7b4df6613412a78a66 +size 221668 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T124053569175Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T124053569175Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc7604c0882dd6d51ee0e0f5dac4abfb20539e09 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T124053569175Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131469076480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_peak": 17298087424, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149491286016, + "reserved_bytes_peak": 149491286016, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:53.568765Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113319511552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_peak": 17298087424, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149491286016, + "reserved_bytes_peak": 149491286016, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:52.636587Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17298087424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149491286016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149491286016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 758, + "segment_count_current_delta": 0, + "segment_count_peak": 758, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T124053569175Z.pickle", + "timestamp_utc": "2026-08-01T12:40:53.575195Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T124053569175Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T124053569175Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba857911ffea25c96327896adf77381e1244e71c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T124053569175Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5737384ebd120bf534ac2b777847471e5afc80422177b26f28e7b843e2c6363 +size 221668 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T124153083488Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T124153083488Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d417e3013df3f6dc1c229ced61efb51df5cb2a40 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T124153083488Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 131469076480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 6996782080, + "inactive_split_bytes_peak": 17298087424, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150384672768, + "reserved_bytes_peak": 150384672768, + "segment_count_current": 764, + "segment_count_peak": 764, + "timestamp_utc": "2026-08-01T12:41:53.082998Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 113319511552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131469076480, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319511552, + "device": 5, + "inactive_split_bytes_current": 11572917248, + "inactive_split_bytes_peak": 17298087424, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149491286016, + "reserved_bytes_peak": 149491286016, + "segment_count_current": 758, + "segment_count_peak": 758, + "timestamp_utc": "2026-08-01T12:40:53.576102Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6996782080, + "inactive_split_bytes_current_delta": -4576135168, + "inactive_split_bytes_peak": 17298087424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150384672768, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150384672768, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 764, + "segment_count_current_delta": 6, + "segment_count_peak": 764, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T124153083488Z.pickle", + "timestamp_utc": "2026-08-01T12:41:53.166368Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T124153083488Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T124153083488Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..faed3771add6f83dcc209380a239574de2915483 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T124153083488Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f3dfe7c3df6813938ecbaa768b8bd0bfb3ba6aadd175e7de06269e8d833e089 +size 3896103 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T124043404631Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T124043404631Z.json new file mode 100644 index 0000000000000000000000000000000000000000..be27363c3829aa9ce13eb44740512d9be281bd66 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T124043404631Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 89675354112, + "active_bytes_peak_delta": 54209017856, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89675354112, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89675354112, + "device": 6, + "inactive_split_bytes_current": 8437941248, + "inactive_split_bytes_peak": 9494840320, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 95693045760, + "reserved_bytes_peak": 95693045760, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.404098Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 89675354112, + "allocated_bytes_peak_delta": 54209017856, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:39:34.005465Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8437941248, + "inactive_split_bytes_current_delta": 8305122304, + "inactive_split_bytes_peak": 9494840320, + "inactive_split_bytes_peak_delta": 9334454272, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 95693045760, + "reserved_bytes_current_delta": 56449040384, + "reserved_bytes_peak": 95693045760, + "reserved_bytes_peak_delta": 56449040384, + "segment_count_current": 467, + "segment_count_current_delta": 396, + "segment_count_peak": 467, + "segment_count_peak_delta": 396, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T124043404631Z.pickle", + "timestamp_utc": "2026-08-01T12:40:43.526024Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T124043404631Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T124043404631Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..19af3d582b835c5c1e2981c84350b1fe2c00dde6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T124043404631Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c64385f5f2a5fb72ae5bc600bbbe98cf4d687a862074ea7a48921aee5d4a754a +size 5411540 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T124044833992Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T124044833992Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dbea9f95dd64fb143ed52c9b2bead4ca1b3fa036 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T124044833992Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113307306496, + "active_bytes_peak_delta": 23631952384, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113307306496, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113307306496, + "device": 6, + "inactive_split_bytes_current": 10409264128, + "inactive_split_bytes_peak": 11240511488, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116001865728, + "reserved_bytes_peak": 116001865728, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:40:44.833576Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113307306496, + "allocated_bytes_peak_delta": 23631952384, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89675354112, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89675354112, + "device": 6, + "inactive_split_bytes_current": 8437941248, + "inactive_split_bytes_peak": 9494840320, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 95693045760, + "reserved_bytes_peak": 95693045760, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.546840Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10409264128, + "inactive_split_bytes_current_delta": 1971322880, + "inactive_split_bytes_peak": 11240511488, + "inactive_split_bytes_peak_delta": 1745671168, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116001865728, + "reserved_bytes_current_delta": 20308819968, + "reserved_bytes_peak": 116001865728, + "reserved_bytes_peak_delta": 20308819968, + "segment_count_current": 534, + "segment_count_current_delta": 67, + "segment_count_peak": 534, + "segment_count_peak_delta": 67, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T124044833992Z.pickle", + "timestamp_utc": "2026-08-01T12:40:44.942847Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T124044833992Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T124044833992Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a9e4e5e5cb4e38ddc4ab883cbf239294d124623 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T124044833992Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9d03b54793695e91454ba55436bb6d04e4b06a07d5092a4bcaf4ff8f2fe3b18 +size 5567636 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T124045967003Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T124045967003Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e1da7126d81581c954b9e429b09f1f44aa91896c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T124045967003Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113307306496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113307306496, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113307306496, + "device": 6, + "inactive_split_bytes_current": 8536507392, + "inactive_split_bytes_peak": 11240511488, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116001865728, + "reserved_bytes_peak": 116001865728, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:40:45.966595Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113307306496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113307306496, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113307306496, + "device": 6, + "inactive_split_bytes_current": 10409264128, + "inactive_split_bytes_peak": 11240511488, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116001865728, + "reserved_bytes_peak": 116001865728, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:40:44.957209Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8536507392, + "inactive_split_bytes_current_delta": -1872756736, + "inactive_split_bytes_peak": 11240511488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116001865728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116001865728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 534, + "segment_count_current_delta": 0, + "segment_count_peak": 534, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T124045967003Z.pickle", + "timestamp_utc": "2026-08-01T12:40:46.083403Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T124045967003Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T124045967003Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..199d7a387d90b1f1d57c5c87aed91b56f37cd87e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T124045967003Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc86714429dcb58fdabab4507fbcbc06e7f2d86ec1a522db2d7e7552381550f1 +size 5573253 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T124048704548Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T124048704548Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c914fff93f20605812917bbd287d55756d6bb28 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T124048704548Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131477989376, + "active_bytes_peak_delta": 18170682880, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_peak": 17386710528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 148386086912, + "reserved_bytes_peak": 148386086912, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:40:48.704034Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113327113728, + "allocated_bytes_peak_delta": 19807232, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113307306496, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113307306496, + "device": 6, + "inactive_split_bytes_current": 8536507392, + "inactive_split_bytes_peak": 11240511488, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116001865728, + "reserved_bytes_peak": 116001865728, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T12:40:46.105000Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_current_delta": 3747344384, + "inactive_split_bytes_peak": 17386710528, + "inactive_split_bytes_peak_delta": 6146199040, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 148386086912, + "reserved_bytes_current_delta": 32384221184, + "reserved_bytes_peak": 148386086912, + "reserved_bytes_peak_delta": 32384221184, + "segment_count_current": 749, + "segment_count_current_delta": 215, + "segment_count_peak": 749, + "segment_count_peak_delta": 215, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T124048704548Z.pickle", + "timestamp_utc": "2026-08-01T12:40:48.820370Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T124048704548Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T124048704548Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8dbd03e77d663bbd59a40f8cdfd9b493df0ad3a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T124048704548Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a3e26e795b2b1c1f49c8f3fb1627d3274fb8c6f369e3664084f1681620cbede +size 5873965 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T124049788483Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T124049788483Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3cb3c265c6028a7d09fc6be2fad84f8aaa8514b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T124049788483Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131477989376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_peak": 17386710528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 148386086912, + "reserved_bytes_peak": 148386086912, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:40:49.788012Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113327113728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_peak": 17386710528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 148386086912, + "reserved_bytes_peak": 148386086912, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:40:48.833664Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17386710528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 148386086912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148386086912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T124049788483Z.pickle", + "timestamp_utc": "2026-08-01T12:40:49.795178Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T124049788483Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T124049788483Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..39c064247cdaecf6628d0e2d98a966f44d9b1a4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T124049788483Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05a4723aad63d690a0dbd6b4b055b2edbf0022c73b80e19a2c059890af45a082 +size 220917 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T124050737643Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T124050737643Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2867cc61d2f7fb1fbbedf46290f3b80f2a17584e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T124050737643Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131477989376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_peak": 17386710528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 148386086912, + "reserved_bytes_peak": 148386086912, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:40:50.737233Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113327113728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_peak": 17386710528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 148386086912, + "reserved_bytes_peak": 148386086912, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:40:49.796155Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17386710528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 148386086912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148386086912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T124050737643Z.pickle", + "timestamp_utc": "2026-08-01T12:40:50.743829Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T124050737643Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T124050737643Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..17725897db9da8398bb7ed73777bbdd17d5498da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T124050737643Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54582ea84da434042c76374c8d001a39cb8682c227184cd2881122b9154cb708 +size 220917 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T124051686948Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T124051686948Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1bd8ce019442b415186608e1c7065a10aa529846 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T124051686948Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131477989376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_peak": 17386710528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 148386086912, + "reserved_bytes_peak": 148386086912, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:40:51.686399Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113327113728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_peak": 17386710528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 148386086912, + "reserved_bytes_peak": 148386086912, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:40:50.744765Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17386710528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 148386086912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148386086912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T124051686948Z.pickle", + "timestamp_utc": "2026-08-01T12:40:51.693714Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T124051686948Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T124051686948Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..734b3261be880834a2a4a39fc64b0da42dbed01b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T124051686948Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ace6c2ace722170cfa692e55b3297eaf9147bd4e26afb36f7b06e0378a0e259a +size 220917 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T124052629404Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T124052629404Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2c999f5b38924486de8f29cf97d757d9662732d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T124052629404Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131477989376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_peak": 17386710528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 148386086912, + "reserved_bytes_peak": 148386086912, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:40:52.629002Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113327113728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_peak": 17386710528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 148386086912, + "reserved_bytes_peak": 148386086912, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:40:51.694736Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17386710528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 148386086912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148386086912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T124052629404Z.pickle", + "timestamp_utc": "2026-08-01T12:40:52.635212Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T124052629404Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T124052629404Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f36d63f5cd8ae9ad6eb0c85027ec1f95487c50d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T124052629404Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e73af1b1aec4d21946b6538af56917655da88fa800ecad9e3cc766e37820cff4 +size 220917 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T124053569231Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T124053569231Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9e1ca5dc4492a4537b346644cf888713ea4e3a82 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T124053569231Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131477989376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_peak": 17386710528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 148386086912, + "reserved_bytes_peak": 148386086912, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:40:53.568768Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113327113728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_peak": 17386710528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 148386086912, + "reserved_bytes_peak": 148386086912, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:40:52.636120Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17386710528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 148386086912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148386086912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 749, + "segment_count_current_delta": 0, + "segment_count_peak": 749, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T124053569231Z.pickle", + "timestamp_utc": "2026-08-01T12:40:53.575906Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T124053569231Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T124053569231Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..61f4a14552c6c6346b692d18df6764c8303dec37 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T124053569231Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:828c9dd81a297410796fa30e6e1ae5c0131cc24c6d443e7b5126ff89f8aa2bcf +size 220917 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T124153083469Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T124153083469Z.json new file mode 100644 index 0000000000000000000000000000000000000000..12824579b75a9aa4605a659b978e7e5869d7fcff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T124153083469Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 131477989376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 7118416896, + "inactive_split_bytes_peak": 17386710528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 149279473664, + "reserved_bytes_peak": 149279473664, + "segment_count_current": 755, + "segment_count_peak": 755, + "timestamp_utc": "2026-08-01T12:41:53.082996Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 113327113728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 131477989376, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113327113728, + "device": 6, + "inactive_split_bytes_current": 12283851776, + "inactive_split_bytes_peak": 17386710528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 148386086912, + "reserved_bytes_peak": 148386086912, + "segment_count_current": 749, + "segment_count_peak": 749, + "timestamp_utc": "2026-08-01T12:40:53.576864Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7118416896, + "inactive_split_bytes_current_delta": -5165434880, + "inactive_split_bytes_peak": 17386710528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 149279473664, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 149279473664, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 755, + "segment_count_current_delta": 6, + "segment_count_peak": 755, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T124153083469Z.pickle", + "timestamp_utc": "2026-08-01T12:41:53.175696Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T124153083469Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T124153083469Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1f9c8f681b5dfcf17130f8666d2b8dc9f2cc09c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T124153083469Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aafcc129daf4c1663d0ed6bd3574b0d39864f6dab0569f72bd329fdd8aaea953 +size 3969866 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T124043404338Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T124043404338Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a4137eb1f21002d5462810260d00a4a838c698df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T124043404338Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 31689664000, + "active_bytes_peak": 89675354112, + "active_bytes_peak_delta": 54209017856, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 89675354112, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 89675354112, + "device": 7, + "inactive_split_bytes_current": 8437940736, + "inactive_split_bytes_peak": 9494840320, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 95693045760, + "reserved_bytes_peak": 95693045760, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.403852Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 31689664000, + "allocated_bytes_peak": 89675354112, + "allocated_bytes_peak_delta": 54209017856, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:39:34.009879Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8437940736, + "inactive_split_bytes_current_delta": 8305121792, + "inactive_split_bytes_peak": 9494840320, + "inactive_split_bytes_peak_delta": 9334454272, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 95693045760, + "reserved_bytes_current_delta": 56449040384, + "reserved_bytes_peak": 95693045760, + "reserved_bytes_peak_delta": 56449040384, + "segment_count_current": 467, + "segment_count_current_delta": 396, + "segment_count_peak": 467, + "segment_count_peak_delta": 396, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T124043404338Z.pickle", + "timestamp_utc": "2026-08-01T12:40:43.518036Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T124043404338Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T124043404338Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c5fade7cd60b4a6c8f7b39a34d90d1282af5700d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T124043404338Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77dfd9bdd9042407bbcc57aee0645814107f38bc0a8a355456ac575d71796876 +size 5413040 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T124044834105Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T124044834105Z.json new file mode 100644 index 0000000000000000000000000000000000000000..461ff13df71e5fa4ce4daa96119f6b7c6a66ed2f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T124044834105Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113303072256, + "active_bytes_peak_delta": 23627718144, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 113303072256, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 113303072256, + "device": 7, + "inactive_split_bytes_current": 10505732608, + "inactive_split_bytes_peak": 10680844800, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116119306240, + "reserved_bytes_peak": 116119306240, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:40:44.833646Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113303072256, + "allocated_bytes_peak_delta": 23627718144, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 89675354112, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 89675354112, + "device": 7, + "inactive_split_bytes_current": 8437940736, + "inactive_split_bytes_peak": 9494840320, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 95693045760, + "reserved_bytes_peak": 95693045760, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T12:40:43.536869Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10505732608, + "inactive_split_bytes_current_delta": 2067791872, + "inactive_split_bytes_peak": 10680844800, + "inactive_split_bytes_peak_delta": 1186004480, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116119306240, + "reserved_bytes_current_delta": 20426260480, + "reserved_bytes_peak": 116119306240, + "reserved_bytes_peak_delta": 20426260480, + "segment_count_current": 535, + "segment_count_current_delta": 68, + "segment_count_peak": 535, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T124044834105Z.pickle", + "timestamp_utc": "2026-08-01T12:40:44.940316Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T124044834105Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T124044834105Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3eed1283cf0403b1119f9c1c6601cb15ec69576c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T124044834105Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f0e4b93e3c1f4cac2b93eff07f4618f9958cab7db69297fee14f94a1b20fda4 +size 5570049 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T124045967102Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T124045967102Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b2557c90281b45eacccf3652cb29f3aa2a3f54f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T124045967102Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113310391296, + "active_bytes_peak_delta": 7319040, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 113310391296, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 113310391296, + "device": 7, + "inactive_split_bytes_current": 9096446464, + "inactive_split_bytes_peak": 10680844800, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116119306240, + "reserved_bytes_peak": 116119306240, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:40:45.966701Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113310391296, + "allocated_bytes_peak_delta": 7319040, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 113303072256, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 113303072256, + "device": 7, + "inactive_split_bytes_current": 10505732608, + "inactive_split_bytes_peak": 10680844800, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116119306240, + "reserved_bytes_peak": 116119306240, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:40:44.955707Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9096446464, + "inactive_split_bytes_current_delta": -1409286144, + "inactive_split_bytes_peak": 10680844800, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116119306240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116119306240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T124045967102Z.pickle", + "timestamp_utc": "2026-08-01T12:40:46.076250Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T124045967102Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T124045967102Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e0567d19ed3b421e593b011c52c2827fe838b62 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T124045967102Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b39f87c083b4d649fd6451f68212786a270c8ed210af96397adc294dbc1f9641 +size 5574447 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T124048704664Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T124048704664Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad19abd1673eddcf4bc834ff91a63f53d39505a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T124048704664Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131474844160, + "active_bytes_peak_delta": 18164452864, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_peak": 17786086912, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149151547392, + "reserved_bytes_peak": 149151547392, + "segment_count_current": 757, + "segment_count_peak": 757, + "timestamp_utc": "2026-08-01T12:40:48.704125Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113324886016, + "allocated_bytes_peak_delta": 14494720, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 113310391296, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 113310391296, + "device": 7, + "inactive_split_bytes_current": 9096446464, + "inactive_split_bytes_peak": 10680844800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116119306240, + "reserved_bytes_peak": 116119306240, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T12:40:46.093532Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_current_delta": 4208717824, + "inactive_split_bytes_peak": 17786086912, + "inactive_split_bytes_peak_delta": 7105242112, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149151547392, + "reserved_bytes_current_delta": 33032241152, + "reserved_bytes_peak": 149151547392, + "reserved_bytes_peak_delta": 33032241152, + "segment_count_current": 757, + "segment_count_current_delta": 222, + "segment_count_peak": 757, + "segment_count_peak_delta": 222, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T124048704664Z.pickle", + "timestamp_utc": "2026-08-01T12:40:48.821323Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T124048704664Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T124048704664Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f85e3cbfa3d6077846f131cb4859e722f328d904 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T124048704664Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f071759314d14cb86841b57776f1325b69a8b233dddf7b9beffad6f67483b91 +size 5871174 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T124049788629Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T124049788629Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a87289328b6f6a90f6600529ec485b366ae72d6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T124049788629Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131474844160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_peak": 17786086912, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149151547392, + "reserved_bytes_peak": 149151547392, + "segment_count_current": 757, + "segment_count_peak": 757, + "timestamp_utc": "2026-08-01T12:40:49.788133Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113324886016, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_peak": 17786086912, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149151547392, + "reserved_bytes_peak": 149151547392, + "segment_count_current": 757, + "segment_count_peak": 757, + "timestamp_utc": "2026-08-01T12:40:48.837497Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17786086912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149151547392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149151547392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 757, + "segment_count_current_delta": 0, + "segment_count_peak": 757, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T124049788629Z.pickle", + "timestamp_utc": "2026-08-01T12:40:49.795771Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T124049788629Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T124049788629Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..30788e893a92c2caddd23a52d5ef3e36eb462c82 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T124049788629Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21821224373e29cdfcd7a7984d5a0e57e75b05b09cbf8a45e51afd85d34ca0ab +size 223826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T124050737714Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T124050737714Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6b55e923d57fd0aa2a62f1d5231c9e0f4f375e80 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T124050737714Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131474844160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_peak": 17786086912, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149151547392, + "reserved_bytes_peak": 149151547392, + "segment_count_current": 757, + "segment_count_peak": 757, + "timestamp_utc": "2026-08-01T12:40:50.737303Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113324886016, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_peak": 17786086912, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149151547392, + "reserved_bytes_peak": 149151547392, + "segment_count_current": 757, + "segment_count_peak": 757, + "timestamp_utc": "2026-08-01T12:40:49.796745Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17786086912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149151547392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149151547392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 757, + "segment_count_current_delta": 0, + "segment_count_peak": 757, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T124050737714Z.pickle", + "timestamp_utc": "2026-08-01T12:40:50.747953Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T124050737714Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T124050737714Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e9345c17f35a43b6da2be1aa10db97035b89eed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T124050737714Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b267f900ab1d5ca03f21ced53aa11507b663b338958b7d7ccf9ef68c50324945 +size 223826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T124051686854Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T124051686854Z.json new file mode 100644 index 0000000000000000000000000000000000000000..be786e956ebd0b55365dd1281960922a26d28931 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T124051686854Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131474844160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_peak": 17786086912, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149151547392, + "reserved_bytes_peak": 149151547392, + "segment_count_current": 757, + "segment_count_peak": 757, + "timestamp_utc": "2026-08-01T12:40:51.686421Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113324886016, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_peak": 17786086912, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149151547392, + "reserved_bytes_peak": 149151547392, + "segment_count_current": 757, + "segment_count_peak": 757, + "timestamp_utc": "2026-08-01T12:40:50.748897Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17786086912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149151547392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149151547392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 757, + "segment_count_current_delta": 0, + "segment_count_peak": 757, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T124051686854Z.pickle", + "timestamp_utc": "2026-08-01T12:40:51.693314Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T124051686854Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T124051686854Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b7731f8638cf5df70e7b1c88d79028cbabc22ea1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T124051686854Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94c23f3389e42b76a157932ede3bcda89824b6eeaebe97842d08da97e6d8bb7b +size 223795 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T124052629494Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T124052629494Z.json new file mode 100644 index 0000000000000000000000000000000000000000..385d4bb91048f8a0a7a740022b3dd6c48149d4af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T124052629494Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131474844160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_peak": 17786086912, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149151547392, + "reserved_bytes_peak": 149151547392, + "segment_count_current": 757, + "segment_count_peak": 757, + "timestamp_utc": "2026-08-01T12:40:52.629082Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113324886016, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_peak": 17786086912, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149151547392, + "reserved_bytes_peak": 149151547392, + "segment_count_current": 757, + "segment_count_peak": 757, + "timestamp_utc": "2026-08-01T12:40:51.694318Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17786086912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149151547392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149151547392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 757, + "segment_count_current_delta": 0, + "segment_count_peak": 757, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T124052629494Z.pickle", + "timestamp_utc": "2026-08-01T12:40:52.635559Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T124052629494Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T124052629494Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c9e8e389189f8d324e93e202cfe610f6a5ccdf3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T124052629494Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9f38fae5293411e7b5ad7f3adcfdd09f540718c99c36685b0f534dd440d1c10 +size 223826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T124053569261Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T124053569261Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe3abd18ba246ae375bce4838bbdd33292a1a4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T124053569261Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131474844160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_peak": 17786086912, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149151547392, + "reserved_bytes_peak": 149151547392, + "segment_count_current": 757, + "segment_count_peak": 757, + "timestamp_utc": "2026-08-01T12:40:53.568823Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113324886016, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_peak": 17786086912, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149151547392, + "reserved_bytes_peak": 149151547392, + "segment_count_current": 757, + "segment_count_peak": 757, + "timestamp_utc": "2026-08-01T12:40:52.636474Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17786086912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149151547392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149151547392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 757, + "segment_count_current_delta": 0, + "segment_count_peak": 757, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T124053569261Z.pickle", + "timestamp_utc": "2026-08-01T12:40:53.575697Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T124053569261Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T124053569261Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..01e269f9e897c10027b1575bad09da82a08e7c7e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T124053569261Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c55bc53526c0e1df37c8de68e5d7540ad2f5ca8224b886b90146eec86b4c82c +size 223826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T124153083491Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T124153083491Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aaf9ad191e8c55ff03c0e1988b9ac1a246932fcd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T124153083491Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955089408, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 131474844160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955089408, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66955089408, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 7529458176, + "inactive_split_bytes_peak": 17786086912, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 150044934144, + "reserved_bytes_peak": 150044934144, + "segment_count_current": 763, + "segment_count_peak": 763, + "timestamp_utc": "2026-08-01T12:41:53.083009Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955089408, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 113324886016, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 131474844160, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 113324886016, + "device": 7, + "inactive_split_bytes_current": 13305164288, + "inactive_split_bytes_peak": 17786086912, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149151547392, + "reserved_bytes_peak": 149151547392, + "segment_count_current": 757, + "segment_count_peak": 757, + "timestamp_utc": "2026-08-01T12:40:53.576648Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7529458176, + "inactive_split_bytes_current_delta": -5775706112, + "inactive_split_bytes_peak": 17786086912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 150044934144, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150044934144, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 763, + "segment_count_current_delta": 6, + "segment_count_peak": 763, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T124153083491Z.pickle", + "timestamp_utc": "2026-08-01T12:41:53.170490Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T124153083491Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T124153083491Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..45dbc753eee6671190ed4a50bd04ba4c49e2d6c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T124153083491Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b8ae1eb8d185468c12b54a8324cf04719f0a51f09c72fe6dd81725e49361f79 +size 3972446 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..ca27a599200ce5d7c9b40194211843922847200f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 4096 +capacity: 1.25 +expert_dispatch_overlap: off +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6017 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 240 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T124448034533Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T124448034533Z.json new file mode 100644 index 0000000000000000000000000000000000000000..892e3b7a47a698b42c937abda37fb0a399f27d2d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T124448034533Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509176832, + "active_bytes_current_delta": 37277721600, + "active_bytes_peak": 89581322240, + "active_bytes_peak_delta": 54114985984, + "after": { + "active_bytes_current": 72509176832, + "active_bytes_peak": 89581322240, + "allocated_bytes_current": 72509176832, + "allocated_bytes_peak": 89581322240, + "device": 0, + "inactive_split_bytes_current": 7589446656, + "inactive_split_bytes_peak": 11216624128, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98283028480, + "reserved_bytes_peak": 98283028480, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T12:44:48.033882Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509176832, + "allocated_bytes_current_delta": 37277721600, + "allocated_bytes_peak": 89581322240, + "allocated_bytes_peak_delta": 54114985984, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:43:39.648490Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7589446656, + "inactive_split_bytes_current_delta": 7454530560, + "inactive_split_bytes_peak": 11216624128, + "inactive_split_bytes_peak_delta": 11055992320, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98283028480, + "reserved_bytes_current_delta": 59036925952, + "reserved_bytes_peak": 98283028480, + "reserved_bytes_peak_delta": 59036925952, + "segment_count_current": 415, + "segment_count_current_delta": 343, + "segment_count_peak": 415, + "segment_count_peak_delta": 343, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T124448034533Z.pickle", + "timestamp_utc": "2026-08-01T12:44:48.158388Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T124448034533Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T124448034533Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f62611d2c9c61256c694a72368123f01b9bc3fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T124448034533Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e512ff3a905985d7dcb1b6f18d57e9d239b67939e49d13ca7a81ab615220e15a +size 7093800 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T124449365067Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T124449365067Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2040144f883fd6568c6a94bdfc8a2de7f2676955 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T124449365067Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509176832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113206168576, + "active_bytes_peak_delta": 23624846336, + "after": { + "active_bytes_current": 72509176832, + "active_bytes_peak": 113206168576, + "allocated_bytes_current": 72509176832, + "allocated_bytes_peak": 113206168576, + "device": 0, + "inactive_split_bytes_current": 9174893568, + "inactive_split_bytes_peak": 13160120832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116893155328, + "reserved_bytes_peak": 116893155328, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T12:44:49.364522Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509176832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113206168576, + "allocated_bytes_peak_delta": 23624846336, + "before": { + "active_bytes_current": 72509176832, + "active_bytes_peak": 89581322240, + "allocated_bytes_current": 72509176832, + "allocated_bytes_peak": 89581322240, + "device": 0, + "inactive_split_bytes_current": 7589446656, + "inactive_split_bytes_peak": 11216624128, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98283028480, + "reserved_bytes_peak": 98283028480, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T12:44:48.178962Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9174893568, + "inactive_split_bytes_current_delta": 1585446912, + "inactive_split_bytes_peak": 13160120832, + "inactive_split_bytes_peak_delta": 1943496704, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116893155328, + "reserved_bytes_current_delta": 18610126848, + "reserved_bytes_peak": 116893155328, + "reserved_bytes_peak_delta": 18610126848, + "segment_count_current": 475, + "segment_count_current_delta": 60, + "segment_count_peak": 475, + "segment_count_peak_delta": 60, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T124449365067Z.pickle", + "timestamp_utc": "2026-08-01T12:44:49.486515Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T124449365067Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T124449365067Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ec65cadb66034866d228419a9bcf256372a6ba96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T124449365067Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:148233326d158cbd39c2aedbf27f32d8354ecb8ecaa8e38d4636d9be4c6a0dec +size 5920930 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T124450650353Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T124450650353Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03928696155816351dae2ffb69e6b9eecb22aac8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T124450650353Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509176832, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113212197888, + "active_bytes_peak_delta": 6029312, + "after": { + "active_bytes_current": 72509176832, + "active_bytes_peak": 113212197888, + "allocated_bytes_current": 72509176832, + "allocated_bytes_peak": 113212197888, + "device": 0, + "inactive_split_bytes_current": 8350712832, + "inactive_split_bytes_peak": 14319755776, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116893155328, + "reserved_bytes_peak": 116893155328, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T12:44:50.649839Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509176832, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113212197888, + "allocated_bytes_peak_delta": 6029312, + "before": { + "active_bytes_current": 72509176832, + "active_bytes_peak": 113206168576, + "allocated_bytes_current": 72509176832, + "allocated_bytes_peak": 113206168576, + "device": 0, + "inactive_split_bytes_current": 9174893568, + "inactive_split_bytes_peak": 13160120832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116893155328, + "reserved_bytes_peak": 116893155328, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T12:44:49.502823Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8350712832, + "inactive_split_bytes_current_delta": -824180736, + "inactive_split_bytes_peak": 14319755776, + "inactive_split_bytes_peak_delta": 1159634944, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116893155328, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116893155328, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 475, + "segment_count_current_delta": 0, + "segment_count_peak": 475, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T124450650353Z.pickle", + "timestamp_utc": "2026-08-01T12:44:50.765303Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T124450650353Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T124450650353Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fd34dab259963b2b67d4ae2755cf356c8bd26a48 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T124450650353Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5473f1b3038e8b115ba69915372c2f79e62d864a315f60da6570e426d5231dcb +size 5926381 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T124453584012Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T124453584012Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bca7358640ab0c0715d4d0c97974ae5322c7ca50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T124453584012Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526220288, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131370928128, + "active_bytes_peak_delta": 18158730240, + "after": { + "active_bytes_current": 72526220288, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 72526220288, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_peak": 22375605248, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_peak": 148931346432, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:44:53.583362Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526220288, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113225819648, + "allocated_bytes_peak_delta": 13621760, + "before": { + "active_bytes_current": 72509176832, + "active_bytes_peak": 113212197888, + "allocated_bytes_current": 72509176832, + "allocated_bytes_peak": 113212197888, + "device": 0, + "inactive_split_bytes_current": 8350712832, + "inactive_split_bytes_peak": 14319755776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116893155328, + "reserved_bytes_peak": 116893155328, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T12:44:50.780156Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_current_delta": 7771779072, + "inactive_split_bytes_peak": 22375605248, + "inactive_split_bytes_peak_delta": 8055849472, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_current_delta": 32038191104, + "reserved_bytes_peak": 148931346432, + "reserved_bytes_peak_delta": 32038191104, + "segment_count_current": 631, + "segment_count_current_delta": 156, + "segment_count_peak": 631, + "segment_count_peak_delta": 156, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T124453584012Z.pickle", + "timestamp_utc": "2026-08-01T12:44:53.713927Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T124453584012Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T124453584012Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e358d4fa50b1df039ed61dbe78c644bce7091578 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T124453584012Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14a3680b275d3354edb68d92c76dbdb83e1e8eb74399eee300e41278e895cc8a +size 6211547 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T124454827255Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T124454827255Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a94cb2d7af2756a9edc0ebf14c777fd1cd8de2e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T124454827255Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526220288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131370928128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526220288, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 72526220288, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_peak": 22375605248, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_peak": 148931346432, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:44:54.826295Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526220288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113225819648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526220288, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 72526220288, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_peak": 22375605248, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_peak": 148931346432, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:44:53.732418Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22375605248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148931346432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 631, + "segment_count_current_delta": 0, + "segment_count_peak": 631, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T124454827255Z.pickle", + "timestamp_utc": "2026-08-01T12:44:54.836174Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T124454827255Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T124454827255Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0edd1b50fc9fd4fd27307b778687976f18482670 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T124454827255Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d997119d903a87845310cd6a6a38f91664d2bb21e19c0966f1d87c8d1f973a0f +size 171485 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T124455906303Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T124455906303Z.json new file mode 100644 index 0000000000000000000000000000000000000000..013c5457cd22a0de0bc14225b3b2a27dcc249782 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T124455906303Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526220288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131370928128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526220288, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 72526220288, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_peak": 22375605248, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_peak": 148931346432, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:44:55.905840Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526220288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113225819648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526220288, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 72526220288, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_peak": 22375605248, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_peak": 148931346432, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:44:54.837386Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22375605248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148931346432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 631, + "segment_count_current_delta": 0, + "segment_count_peak": 631, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T124455906303Z.pickle", + "timestamp_utc": "2026-08-01T12:44:55.911835Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T124455906303Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T124455906303Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..498f991d4da1e91db1c30a52303f10c25824914f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T124455906303Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc0b0377c74632d26b89298cb4dd6d7711326b6114c0f92d1da0219dc8ff23d5 +size 171485 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T124456978184Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T124456978184Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36403c2215e700337cc82c3b5504d234cc1e5a30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T124456978184Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526220288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131370928128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526220288, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 72526220288, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_peak": 22375605248, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_peak": 148931346432, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:44:56.977762Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526220288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113225819648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526220288, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 72526220288, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_peak": 22375605248, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_peak": 148931346432, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:44:55.912736Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22375605248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148931346432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 631, + "segment_count_current_delta": 0, + "segment_count_peak": 631, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T124456978184Z.pickle", + "timestamp_utc": "2026-08-01T12:44:56.982941Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T124456978184Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T124456978184Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe3d0d6e6af87c247b71045360430954448b6577 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T124456978184Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a30c80d4d2ebdd0b5d22546efa5d1c7c4d9b1fdb843e4346d2369585b7807c4d +size 171485 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T124458046024Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T124458046024Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0cf055db1977e9fef432e74e7b7b2b67ef47a93b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T124458046024Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526220288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131370928128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526220288, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 72526220288, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_peak": 22375605248, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_peak": 148931346432, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:44:58.045609Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526220288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113225819648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526220288, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 72526220288, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_peak": 22375605248, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_peak": 148931346432, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:44:56.983829Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22375605248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148931346432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 631, + "segment_count_current_delta": 0, + "segment_count_peak": 631, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T124458046024Z.pickle", + "timestamp_utc": "2026-08-01T12:44:58.050875Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T124458046024Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T124458046024Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df4dbc32ff742a27d908003f3a1972e1e935ddab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T124458046024Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbf519b269df7b6a54c36f645c46b979860617948874b579f4b2a6e19bba0966 +size 171485 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T124459121217Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T124459121217Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bdff7550b492fc3e7e7a1b1d9263ddaefe6d52d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T124459121217Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526220288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131370928128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526220288, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 72526220288, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_peak": 22375605248, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_peak": 148931346432, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:44:59.120755Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526220288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113225819648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526220288, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 72526220288, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_peak": 22375605248, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_peak": 148931346432, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:44:58.051745Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 22375605248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148931346432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 631, + "segment_count_current_delta": 0, + "segment_count_peak": 631, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T124459121217Z.pickle", + "timestamp_utc": "2026-08-01T12:44:59.126268Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T124459121217Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T124459121217Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a0c35127ab6a6f81d4b84098fe639b52b4411972 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T124459121217Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d4c98d52d5746d2c00f9225d3071345e3b1ee9bf093f8465c2ca6c14ece6f19 +size 171485 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T124558843405Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T124558843405Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a74bd2723ff69c89cec33de687dc3ab46b7a27af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T124558843405Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61319500800, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 131370928128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61319500800, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 61319500800, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 9279024128, + "inactive_split_bytes_peak": 22375605248, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 149000552448, + "reserved_bytes_peak": 149000552448, + "segment_count_current": 636, + "segment_count_peak": 636, + "timestamp_utc": "2026-08-01T12:45:58.842904Z", + "world_size": 8 + }, + "allocated_bytes_current": 61319500800, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 113225819648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526220288, + "active_bytes_peak": 131370928128, + "allocated_bytes_current": 72526220288, + "allocated_bytes_peak": 113225819648, + "device": 0, + "inactive_split_bytes_current": 16122491904, + "inactive_split_bytes_peak": 22375605248, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 148931346432, + "reserved_bytes_peak": 148931346432, + "segment_count_current": 631, + "segment_count_peak": 631, + "timestamp_utc": "2026-08-01T12:44:59.127145Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9279024128, + "inactive_split_bytes_current_delta": -6843467776, + "inactive_split_bytes_peak": 22375605248, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 149000552448, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 149000552448, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 636, + "segment_count_current_delta": 5, + "segment_count_peak": 636, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T124558843405Z.pickle", + "timestamp_utc": "2026-08-01T12:45:58.926267Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T124558843405Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T124558843405Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e4f0e7ec1e000bf203c2fdda113fa772848a84f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T124558843405Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82f205d8da33500d59d2ed1d16d2ccfd5087af5f48cda0354c1151f6c2e95213 +size 3922640 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T124448034358Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T124448034358Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d3d4de37977c662982e4e1e01ddd7ace27ed966 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T124448034358Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 37277213696, + "active_bytes_peak": 89570930688, + "active_bytes_peak_delta": 54104594432, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 89570930688, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 89570930688, + "device": 1, + "inactive_split_bytes_current": 7971636224, + "inactive_split_bytes_peak": 10777481216, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 98278834176, + "reserved_bytes_peak": 98278834176, + "segment_count_current": 412, + "segment_count_peak": 412, + "timestamp_utc": "2026-08-01T12:44:48.033728Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 37277213696, + "allocated_bytes_peak": 89570930688, + "allocated_bytes_peak_delta": 54104594432, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:43:39.648103Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7971636224, + "inactive_split_bytes_current_delta": 7838817280, + "inactive_split_bytes_peak": 10777481216, + "inactive_split_bytes_peak_delta": 10617095168, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 98278834176, + "reserved_bytes_current_delta": 59034828800, + "reserved_bytes_peak": 98278834176, + "reserved_bytes_peak_delta": 59034828800, + "segment_count_current": 412, + "segment_count_current_delta": 341, + "segment_count_peak": 412, + "segment_count_peak_delta": 341, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T124448034358Z.pickle", + "timestamp_utc": "2026-08-01T12:44:48.156613Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T124448034358Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T124448034358Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b8705d235a1f902d8b06314e3d9c38019ae1be78 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T124448034358Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0efafd68322cab96e09f4ee706e788a51fc540e11881781ea8f01cc1e7e3cd64 +size 7078232 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T124449364895Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T124449364895Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3b0d3a9059fedc4c8947cb78378a19f6379637e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T124449364895Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113206119424, + "active_bytes_peak_delta": 23635188736, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113206119424, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113206119424, + "device": 1, + "inactive_split_bytes_current": 9022309376, + "inactive_split_bytes_peak": 13022193152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:49.364490Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113206119424, + "allocated_bytes_peak_delta": 23635188736, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 89570930688, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 89570930688, + "device": 1, + "inactive_split_bytes_current": 7971636224, + "inactive_split_bytes_peak": 10777481216, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 98278834176, + "reserved_bytes_peak": 98278834176, + "segment_count_current": 412, + "segment_count_peak": 412, + "timestamp_utc": "2026-08-01T12:44:48.178021Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9022309376, + "inactive_split_bytes_current_delta": 1050673152, + "inactive_split_bytes_peak": 13022193152, + "inactive_split_bytes_peak_delta": 2244711936, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116821852160, + "reserved_bytes_current_delta": 18543017984, + "reserved_bytes_peak": 116821852160, + "reserved_bytes_peak_delta": 18543017984, + "segment_count_current": 474, + "segment_count_current_delta": 62, + "segment_count_peak": 474, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T124449364895Z.pickle", + "timestamp_utc": "2026-08-01T12:44:49.481245Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T124449364895Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T124449364895Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ed39e148ee7915c5b240567c67c3e9d4d8b1a35 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T124449364895Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd29b5f51e40aaedb0613c3246e177eb72772d9f83014aea60029d1a7ac8fc04 +size 5913419 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T124450650189Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T124450650189Z.json new file mode 100644 index 0000000000000000000000000000000000000000..301eeaab166a1a446f157eb2a14581935941e8a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T124450650189Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113208388608, + "active_bytes_peak_delta": 2269184, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113208388608, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113208388608, + "device": 1, + "inactive_split_bytes_current": 9322202112, + "inactive_split_bytes_peak": 15426136064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:50.649786Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113208388608, + "allocated_bytes_peak_delta": 2269184, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113206119424, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113206119424, + "device": 1, + "inactive_split_bytes_current": 9022309376, + "inactive_split_bytes_peak": 13022193152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:49.498884Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9322202112, + "inactive_split_bytes_current_delta": 299892736, + "inactive_split_bytes_peak": 15426136064, + "inactive_split_bytes_peak_delta": 2403942912, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116821852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116821852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T124450650189Z.pickle", + "timestamp_utc": "2026-08-01T12:44:50.762292Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T124450650189Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T124450650189Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8f67b00808dfca77edc56b1fd6a8524d0f97d376 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T124450650189Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08f5e235d869e09b54baeaa63efdc60cda3e7ce2b94f87b3c0d1d099476f8381 +size 5918396 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T124453583767Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T124453583767Z.json new file mode 100644 index 0000000000000000000000000000000000000000..081dcfd56ae241f38254b8e68f90a91f4ae4bc24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T124453583767Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 18163080192, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:53.583226Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 12728832, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113208388608, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113208388608, + "device": 1, + "inactive_split_bytes_current": 9322202112, + "inactive_split_bytes_peak": 15426136064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:50.779193Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_current_delta": 7566258176, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 6249790464, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 32768000000, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 32768000000, + "segment_count_current": 623, + "segment_count_current_delta": 149, + "segment_count_peak": 623, + "segment_count_peak_delta": 149, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T124453583767Z.pickle", + "timestamp_utc": "2026-08-01T12:44:53.708309Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T124453583767Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T124453583767Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fa56dedbcbd7faf228a41fff065bcbf097ffe323 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T124453583767Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a174df1b30ac0bc22ff9d48d39d109df2724ef3c5736b4208a26175b5ad60eb7 +size 6202067 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T124454826972Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T124454826972Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5861c516eff44235569c00532097af09d98b4229 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T124454826972Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:54.825991Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:53.725637Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T124454826972Z.pickle", + "timestamp_utc": "2026-08-01T12:44:54.833477Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T124454826972Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T124454826972Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b172d628ba2eab1f8e5360dae343b66e4ffa572 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T124454826972Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdbea72ced7daf4fe44128e9c10a1661352fcd2d0fdf84c2aee46cd2ec23c1aa +size 163469 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T124455906258Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T124455906258Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a87796d6c557b85e55506c15aa3ed01167e5caf8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T124455906258Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:55.905804Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:54.834489Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T124455906258Z.pickle", + "timestamp_utc": "2026-08-01T12:44:55.911173Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T124455906258Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T124455906258Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f6c39cc9bdeb32fd2b2452eaa55f17e3df9f7ab4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T124455906258Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39cf067fd7cfa0b31c962652065e7acfe0d45e2dab0ffa3a0cfc0ad296b2d46c +size 163438 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T124456978221Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T124456978221Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e0db15e8dff88e292cda3d14904ff56e2271e0d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T124456978221Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:56.977790Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:55.912083Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T124456978221Z.pickle", + "timestamp_utc": "2026-08-01T12:44:56.982940Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T124456978221Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T124456978221Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0d8faab32ff2c9a5a289c67929afc9827c02d14a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T124456978221Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b9b4cbdaf5e7d25144c89e258192c718fc5237872e15fda8a941fe5a144518a +size 163469 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T124458046042Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T124458046042Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd71dc67583e783cfa1a51291546d40d91302e30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T124458046042Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:58.045626Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:56.983854Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T124458046042Z.pickle", + "timestamp_utc": "2026-08-01T12:44:58.050734Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T124458046042Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T124458046042Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e2025a6aca6f91b525ec8f4c81230bda8b148aae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T124458046042Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4591d403605d427fd2241a637307c900ad0897f2306f1000ea07d30e92b2f804 +size 163438 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T124459121336Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T124459121336Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c5532c676250ab6e83f81b8bba245a4757f723d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T124459121336Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:59.120791Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:58.051650Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T124459121336Z.pickle", + "timestamp_utc": "2026-08-01T12:44:59.126750Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T124459121336Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T124459121336Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..770d5c0f8cafd0bbe29058e4273235562e807017 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T124459121336Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef900b6341c66ea081986110bc3faf1f45434a9bdaa9b6437d43d970d7912f8e +size 163469 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T124558843407Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T124558843407Z.json new file mode 100644 index 0000000000000000000000000000000000000000..12942093b7d5ce81fc251180b9b6f71a965903cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T124558843407Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61318992896, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61318992896, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 61318992896, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 11187940352, + "inactive_split_bytes_peak": 21675926528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 150483238912, + "reserved_bytes_peak": 150483238912, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:45:58.842889Z", + "world_size": 8 + }, + "allocated_bytes_current": 61318992896, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 1, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:59.127645Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11187940352, + "inactive_split_bytes_current_delta": -5700519936, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 150483238912, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150483238912, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 629, + "segment_count_current_delta": 6, + "segment_count_peak": 629, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T124558843407Z.pickle", + "timestamp_utc": "2026-08-01T12:45:58.928178Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T124558843407Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T124558843407Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca38f914cc5257ca25899ace45b81b8b5ae1b03d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T124558843407Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce3e33ab093b7353da701e884ed7f6582934a7943897c21a9d42f744c5aa7b13 +size 3841347 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T124448034315Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T124448034315Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c1a079ed5722c2c3f22a8835641fe2c2af353a64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T124448034315Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 37277213696, + "active_bytes_peak": 89570930688, + "active_bytes_peak_delta": 54104594432, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 89570930688, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 89570930688, + "device": 2, + "inactive_split_bytes_current": 7969539072, + "inactive_split_bytes_peak": 10777481216, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98278834176, + "reserved_bytes_peak": 98278834176, + "segment_count_current": 412, + "segment_count_peak": 412, + "timestamp_utc": "2026-08-01T12:44:48.033729Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 37277213696, + "allocated_bytes_peak": 89570930688, + "allocated_bytes_peak_delta": 54104594432, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:43:39.648153Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7969539072, + "inactive_split_bytes_current_delta": 7836720128, + "inactive_split_bytes_peak": 10777481216, + "inactive_split_bytes_peak_delta": 10617095168, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98278834176, + "reserved_bytes_current_delta": 59034828800, + "reserved_bytes_peak": 98278834176, + "reserved_bytes_peak_delta": 59034828800, + "segment_count_current": 412, + "segment_count_current_delta": 341, + "segment_count_peak": 412, + "segment_count_peak_delta": 341, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T124448034315Z.pickle", + "timestamp_utc": "2026-08-01T12:44:48.152946Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T124448034315Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T124448034315Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..98508985616ccbc3c283a4e97302251a62c4e513 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T124448034315Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44ceaec79ef74b7b46545e00472a496c6bdc14d8b3625094c3e3227b56a9322e +size 7078260 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T124449364889Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T124449364889Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bb5142ec0c0f9f7f0804b70dc118ec2f05d28754 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T124449364889Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113206119424, + "active_bytes_peak_delta": 23635188736, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113206119424, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113206119424, + "device": 2, + "inactive_split_bytes_current": 9028600832, + "inactive_split_bytes_peak": 13022193152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:49.364481Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113206119424, + "allocated_bytes_peak_delta": 23635188736, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 89570930688, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 89570930688, + "device": 2, + "inactive_split_bytes_current": 7969539072, + "inactive_split_bytes_peak": 10777481216, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98278834176, + "reserved_bytes_peak": 98278834176, + "segment_count_current": 412, + "segment_count_peak": 412, + "timestamp_utc": "2026-08-01T12:44:48.173230Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9028600832, + "inactive_split_bytes_current_delta": 1059061760, + "inactive_split_bytes_peak": 13022193152, + "inactive_split_bytes_peak_delta": 2244711936, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116821852160, + "reserved_bytes_current_delta": 18543017984, + "reserved_bytes_peak": 116821852160, + "reserved_bytes_peak_delta": 18543017984, + "segment_count_current": 474, + "segment_count_current_delta": 62, + "segment_count_peak": 474, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T124449364889Z.pickle", + "timestamp_utc": "2026-08-01T12:44:49.480296Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T124449364889Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T124449364889Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3b1e20d3020a19bfaf45e1e95bdcf55cef663cff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T124449364889Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e373a00b738ffde3fa1ace14cac169794c6bed851db4a6e8e41500ad3c3d2a2 +size 5913434 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T124450650201Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T124450650201Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d59d838c816555b1f729a7e09540c38a0cf2023b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T124450650201Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113208388608, + "active_bytes_peak_delta": 2269184, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113208388608, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113208388608, + "device": 2, + "inactive_split_bytes_current": 9322202112, + "inactive_split_bytes_peak": 15426136064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116823949312, + "reserved_bytes_peak": 116823949312, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T12:44:50.649795Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113208388608, + "allocated_bytes_peak_delta": 2269184, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113206119424, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113206119424, + "device": 2, + "inactive_split_bytes_current": 9028600832, + "inactive_split_bytes_peak": 13022193152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:49.497439Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9322202112, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 15426136064, + "inactive_split_bytes_peak_delta": 2403942912, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116823949312, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 116823949312, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 475, + "segment_count_current_delta": 1, + "segment_count_peak": 475, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T124450650201Z.pickle", + "timestamp_utc": "2026-08-01T12:44:50.762131Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T124450650201Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T124450650201Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a26dab5e0c7107d05e52d7981dfbcc089d521f2d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T124450650201Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61997570e5959423f84cb0128f4f36c762e37dfec5970f2fb56f54645d7f3602 +size 5918409 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T124453583735Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T124453583735Z.json new file mode 100644 index 0000000000000000000000000000000000000000..35bce75a2bcc8478a3f6a4b1f691c4f651e2c6ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T124453583735Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 18163080192, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:53.583207Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 12728832, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113208388608, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113208388608, + "device": 2, + "inactive_split_bytes_current": 9322202112, + "inactive_split_bytes_peak": 15426136064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116823949312, + "reserved_bytes_peak": 116823949312, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T12:44:50.778786Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_current_delta": 7566258176, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 6249790464, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 32765902848, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 32765902848, + "segment_count_current": 623, + "segment_count_current_delta": 148, + "segment_count_peak": 623, + "segment_count_peak_delta": 148, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T124453583735Z.pickle", + "timestamp_utc": "2026-08-01T12:44:53.708710Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T124453583735Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T124453583735Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..48f1dfdd7e09353ce1f94ad736aa0c26e78c187f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T124453583735Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:275463d08ab5074954021b4bee27c055ddc9207d242c166f51a2ac47735367ab +size 6202140 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T124454826717Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T124454826717Z.json new file mode 100644 index 0000000000000000000000000000000000000000..365ec4d6f6b0f74062afb01ffe7bdd34c1ccb043 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T124454826717Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:54.825876Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:53.725693Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T124454826717Z.pickle", + "timestamp_utc": "2026-08-01T12:44:54.831473Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T124454826717Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T124454826717Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..17e3399bfbaa93e15dc044ec74c91bd52dd527ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T124454826717Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7252bfa3bab99959b638087cebec03a33cb824ce8a4d014c8ecaf092152bdb92 +size 163428 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T124455906250Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T124455906250Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d504ccb97bbdbb81f3e1635c6fd1fef5aa3e50b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T124455906250Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:55.905792Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:54.832400Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T124455906250Z.pickle", + "timestamp_utc": "2026-08-01T12:44:55.910801Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T124455906250Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T124455906250Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a7d5a6a960f40ef96cc7681752eef2abb48fbd4b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T124455906250Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38ea97c28f58e81aa1c299f371c947d750f5d9128374151773e4cdd67c0a3209 +size 163459 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T124456978201Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T124456978201Z.json new file mode 100644 index 0000000000000000000000000000000000000000..069e030ad9238641e1635163e11460ef19f5c493 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T124456978201Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:56.977772Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:55.911655Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T124456978201Z.pickle", + "timestamp_utc": "2026-08-01T12:44:56.982792Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T124456978201Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T124456978201Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ac560deef665570aa87538df17f01959b68650c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T124456978201Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2da7fe877530c3186738fa0707724969454c0a6537cb5d25b89df5756ffa77a0 +size 163428 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T124458046018Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T124458046018Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5dfec347711b1d78bbde29004f32514db07a8c9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T124458046018Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:58.045606Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:56.983654Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T124458046018Z.pickle", + "timestamp_utc": "2026-08-01T12:44:58.050583Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T124458046018Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T124458046018Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94c7803d5bc1aa749831a3aba9e317babe43b2f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T124458046018Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c28579c901e1d48d0bd06d5d8728db9bc773ff96a02acc422c88c37a500d778 +size 163459 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T124459121178Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T124459121178Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87a5b2f43192329cce43ec5fc412dabbb7bb1c1e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T124459121178Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:59.120749Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:58.051444Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T124459121178Z.pickle", + "timestamp_utc": "2026-08-01T12:44:59.125891Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T124459121178Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T124459121178Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c1cd912621762519c984cbeffbe5d6a9cbf78e9c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T124459121178Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e04735187b3b6044840eb45264fdda210a54adf1c13653ba786a3b4650f736c +size 163428 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T124558843405Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T124558843405Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2cba1aff7254e4b28a9ba8aa42f3f4fbc72f3914 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T124558843405Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61318992896, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61318992896, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 61318992896, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 11187940352, + "inactive_split_bytes_peak": 21675926528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 150483238912, + "reserved_bytes_peak": 150483238912, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:45:58.842901Z", + "world_size": 8 + }, + "allocated_bytes_current": 61318992896, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 2, + "inactive_split_bytes_current": 16888460288, + "inactive_split_bytes_peak": 21675926528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:59.126763Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 11187940352, + "inactive_split_bytes_current_delta": -5700519936, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 150483238912, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150483238912, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 629, + "segment_count_current_delta": 6, + "segment_count_peak": 629, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T124558843405Z.pickle", + "timestamp_utc": "2026-08-01T12:45:58.926187Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T124558843405Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T124558843405Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e5921606fdeb16e403618ba1554a1ed326670eec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T124558843405Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e2f0a19b5337d23170f3f44096874e582471644dad0438643aa593c59bc3fe6 +size 3841464 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T124448034876Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T124448034876Z.json new file mode 100644 index 0000000000000000000000000000000000000000..12f23ebfa0504d742f1c4ea8be72998336e1c82d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T124448034876Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 37277213696, + "active_bytes_peak": 89570930688, + "active_bytes_peak_delta": 54104594432, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 89570930688, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 89570930688, + "device": 3, + "inactive_split_bytes_current": 7969539072, + "inactive_split_bytes_peak": 10777481216, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 98278834176, + "reserved_bytes_peak": 98278834176, + "segment_count_current": 412, + "segment_count_peak": 412, + "timestamp_utc": "2026-08-01T12:44:48.034048Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 37277213696, + "allocated_bytes_peak": 89570930688, + "allocated_bytes_peak_delta": 54104594432, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:43:39.648232Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7969539072, + "inactive_split_bytes_current_delta": 7836720128, + "inactive_split_bytes_peak": 10777481216, + "inactive_split_bytes_peak_delta": 10617095168, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 98278834176, + "reserved_bytes_current_delta": 59034828800, + "reserved_bytes_peak": 98278834176, + "reserved_bytes_peak_delta": 59034828800, + "segment_count_current": 412, + "segment_count_current_delta": 341, + "segment_count_peak": 412, + "segment_count_peak_delta": 341, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T124448034876Z.pickle", + "timestamp_utc": "2026-08-01T12:44:48.157479Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T124448034876Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T124448034876Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8b446a945967acf0fa95864a0e8b21270b92ace5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T124448034876Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e7ecb9168836d576c1bd2ad614e45c0d5a120243342fd8bf9914c043fc3d432 +size 7078260 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T124449365008Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T124449365008Z.json new file mode 100644 index 0000000000000000000000000000000000000000..870c48d4e160c793ea445ecc42f13fbad2ce8701 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T124449365008Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113206119424, + "active_bytes_peak_delta": 23635188736, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113206119424, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113206119424, + "device": 3, + "inactive_split_bytes_current": 9024406528, + "inactive_split_bytes_peak": 13022193152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:49.364583Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113206119424, + "allocated_bytes_peak_delta": 23635188736, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 89570930688, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 89570930688, + "device": 3, + "inactive_split_bytes_current": 7969539072, + "inactive_split_bytes_peak": 10777481216, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 98278834176, + "reserved_bytes_peak": 98278834176, + "segment_count_current": 412, + "segment_count_peak": 412, + "timestamp_utc": "2026-08-01T12:44:48.174397Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9024406528, + "inactive_split_bytes_current_delta": 1054867456, + "inactive_split_bytes_peak": 13022193152, + "inactive_split_bytes_peak_delta": 2244711936, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116821852160, + "reserved_bytes_current_delta": 18543017984, + "reserved_bytes_peak": 116821852160, + "reserved_bytes_peak_delta": 18543017984, + "segment_count_current": 474, + "segment_count_current_delta": 62, + "segment_count_peak": 474, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T124449365008Z.pickle", + "timestamp_utc": "2026-08-01T12:44:49.476985Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T124449365008Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T124449365008Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fd43e36362e1de50f8b2317fe951ab3bb3c4a5fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T124449365008Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0b6043f3e7edfe59d66165c09d6770d7d6608e27f1d9b4ea07b1fec73353678 +size 5913385 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T124450650328Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T124450650328Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e7e24472efb02b67b2d8657d2a34773b3503b394 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T124450650328Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113208388608, + "active_bytes_peak_delta": 2269184, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113208388608, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113208388608, + "device": 3, + "inactive_split_bytes_current": 9322202112, + "inactive_split_bytes_peak": 15426144256, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:50.649898Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113208388608, + "allocated_bytes_peak_delta": 2269184, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113206119424, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113206119424, + "device": 3, + "inactive_split_bytes_current": 9024406528, + "inactive_split_bytes_peak": 13022193152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:49.491201Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9322202112, + "inactive_split_bytes_current_delta": 297795584, + "inactive_split_bytes_peak": 15426144256, + "inactive_split_bytes_peak_delta": 2403951104, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116821852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116821852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T124450650328Z.pickle", + "timestamp_utc": "2026-08-01T12:44:50.758138Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T124450650328Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T124450650328Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a694ac0d9c7caf5f995e18b5c34fae8a5fe303da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T124450650328Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd8df6b552b50b85ddb96493291ad7a084c57cc02fe23caa2607ce77359a4126 +size 5918386 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T124453584274Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T124453584274Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f4b788cba5433cc7247fde5eaf1132bb428f933 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T124453584274Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 18163080192, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:53.583629Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 12728832, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113208388608, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113208388608, + "device": 3, + "inactive_split_bytes_current": 9322202112, + "inactive_split_bytes_peak": 15426144256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:50.771798Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 7564161024, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 6249782272, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 32768000000, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 32768000000, + "segment_count_current": 623, + "segment_count_current_delta": 149, + "segment_count_peak": 623, + "segment_count_peak_delta": 149, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T124453584274Z.pickle", + "timestamp_utc": "2026-08-01T12:44:53.723895Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T124453584274Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T124453584274Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..83b1e81721ab920220f16b6c0d6fd1eedf5da610 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T124453584274Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2a4f5f18e7761ae52f9c5881faf9e6acb21cb646ca6d356770c7962662479b5 +size 6202081 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T124454827035Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T124454827035Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3d50d8dad9d46f7da1f3d991c5d86a5d47e70586 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T124454827035Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:54.825946Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:53.742930Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T124454827035Z.pickle", + "timestamp_utc": "2026-08-01T12:44:54.833462Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T124454827035Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T124454827035Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c43b72d83c2e714f4ba4a43d242c8d3bd6be620e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T124454827035Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dd9d8154826c5279d9526b162d56497dae9fa127013f229470c03f5becfd2c3 +size 163421 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T124455906332Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T124455906332Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d4c3e1e54e446104414def55e875988075b306ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T124455906332Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:55.905844Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:54.834540Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T124455906332Z.pickle", + "timestamp_utc": "2026-08-01T12:44:55.911685Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T124455906332Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T124455906332Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..751f2ddf83cb9dc32454deb694b9fbd0513668a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T124455906332Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ee59cfd99ccea0c577b7c134bd8cd7143594f0ed82a16232b4736bc62e0dbd5 +size 163452 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T124456978630Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T124456978630Z.json new file mode 100644 index 0000000000000000000000000000000000000000..496e5353e8b917a70a7555681231f166a3b678ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T124456978630Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:56.978039Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:55.912580Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T124456978630Z.pickle", + "timestamp_utc": "2026-08-01T12:44:56.985216Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T124456978630Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T124456978630Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a39a6af35932617d23894ce64f5d9c58045c9aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T124456978630Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54da1c7d809b848c48cb4ccdb50122598c8f67c6165655b03654313846733552 +size 163421 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T124458046462Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T124458046462Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b137615dc7df7f26914d30fa26172fcd46f37a21 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T124458046462Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:58.045858Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:56.986271Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T124458046462Z.pickle", + "timestamp_utc": "2026-08-01T12:44:58.052625Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T124458046462Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T124458046462Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9db323b04070990ae1f81abdf15d89a9b6d8b6a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T124458046462Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de07958f2d6075d8bbeaddb39a25199840118bfaa90c223eb3d39d3a8794f7eb +size 163452 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T124459121368Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T124459121368Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3f17d0d19ef96ea8e39f37e071cd740971454e7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T124459121368Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:59.120883Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:58.053680Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T124459121368Z.pickle", + "timestamp_utc": "2026-08-01T12:44:59.126478Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T124459121368Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T124459121368Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9933042688d5850c909d8967ecfb21f62296c54a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T124459121368Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9a8bd2a05b4113e4cedd11ad65affed39b3f4cb81d08fd10580f9bd8202eab3 +size 163421 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T124558843407Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T124558843407Z.json new file mode 100644 index 0000000000000000000000000000000000000000..142b6baa9e990ca2bdee0d0b0003c474be84950a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T124558843407Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61318992896, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61318992896, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 61318992896, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 11185843200, + "inactive_split_bytes_peak": 21675926528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 150483238912, + "reserved_bytes_peak": 150483238912, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:45:58.842911Z", + "world_size": 8 + }, + "allocated_bytes_current": 61318992896, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 3, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:59.127365Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 11185843200, + "inactive_split_bytes_current_delta": -5700519936, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 150483238912, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150483238912, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 629, + "segment_count_current_delta": 6, + "segment_count_peak": 629, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T124558843407Z.pickle", + "timestamp_utc": "2026-08-01T12:45:58.928184Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T124558843407Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T124558843407Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1f9754cf5a336b9237a1dbb34af100fe1662b78d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T124558843407Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a62281a4f5aded2e0a3c307a710b068937fb1f1b1ab7fe99e5c20c6ee918728 +size 3841397 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T124448034368Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T124448034368Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed5718fc97eefb73095646789547bb3ea2317d60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T124448034368Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72510444544, + "active_bytes_current_delta": 37278546944, + "active_bytes_peak": 89566867968, + "active_bytes_peak_delta": 54100089344, + "after": { + "active_bytes_current": 72510444544, + "active_bytes_peak": 89566867968, + "allocated_bytes_current": 72510444544, + "allocated_bytes_peak": 89566867968, + "device": 4, + "inactive_split_bytes_current": 8085203968, + "inactive_split_bytes_peak": 11343715840, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 98574532608, + "reserved_bytes_peak": 98574532608, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T12:44:48.033751Z", + "world_size": 8 + }, + "allocated_bytes_current": 72510444544, + "allocated_bytes_current_delta": 37278546944, + "allocated_bytes_peak": 89566867968, + "allocated_bytes_peak_delta": 54100089344, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:43:39.647263Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8085203968, + "inactive_split_bytes_current_delta": 7952827392, + "inactive_split_bytes_peak": 11343715840, + "inactive_split_bytes_peak_delta": 11190306304, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 98574532608, + "reserved_bytes_current_delta": 59330527232, + "reserved_bytes_peak": 98574532608, + "reserved_bytes_peak_delta": 59330527232, + "segment_count_current": 415, + "segment_count_current_delta": 344, + "segment_count_peak": 415, + "segment_count_peak_delta": 344, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T124448034368Z.pickle", + "timestamp_utc": "2026-08-01T12:44:48.157033Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T124448034368Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T124448034368Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cfff709172cbe0e58c663af3a61c5da16e6d77c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T124448034368Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92f975d98e7b7b68111afaad78e58b08cd5362c0c879a0634c1eb45f0dca454b +size 7102335 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T124449364847Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T124449364847Z.json new file mode 100644 index 0000000000000000000000000000000000000000..86e8489691d066602cd3f915fdb4108abba3a95f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T124449364847Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72510444544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113203280896, + "active_bytes_peak_delta": 23636412928, + "after": { + "active_bytes_current": 72510444544, + "active_bytes_peak": 113203280896, + "allocated_bytes_current": 72510444544, + "allocated_bytes_peak": 113203280896, + "device": 4, + "inactive_split_bytes_current": 8672406528, + "inactive_split_bytes_peak": 14187800064, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116685537280, + "reserved_bytes_peak": 116685537280, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:44:49.364438Z", + "world_size": 8 + }, + "allocated_bytes_current": 72510444544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113203280896, + "allocated_bytes_peak_delta": 23636412928, + "before": { + "active_bytes_current": 72510444544, + "active_bytes_peak": 89566867968, + "allocated_bytes_current": 72510444544, + "allocated_bytes_peak": 89566867968, + "device": 4, + "inactive_split_bytes_current": 8085203968, + "inactive_split_bytes_peak": 11343715840, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 98574532608, + "reserved_bytes_peak": 98574532608, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T12:44:48.178409Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8672406528, + "inactive_split_bytes_current_delta": 587202560, + "inactive_split_bytes_peak": 14187800064, + "inactive_split_bytes_peak_delta": 2844084224, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116685537280, + "reserved_bytes_current_delta": 18111004672, + "reserved_bytes_peak": 116685537280, + "reserved_bytes_peak_delta": 18111004672, + "segment_count_current": 468, + "segment_count_current_delta": 53, + "segment_count_peak": 468, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T124449364847Z.pickle", + "timestamp_utc": "2026-08-01T12:44:49.481819Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T124449364847Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T124449364847Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..331e6d14fd355f69943f2bfd07e3a3d6806a2a8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T124449364847Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:226a8287c0cd7efd7ba9e828370be61205d4bb19df7545ad216308ab99450651 +size 5919521 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T124450650177Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T124450650177Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2374e2b20d98ed90622f67e95330c3b5504aa9d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T124450650177Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72510444544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113203280896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72510444544, + "active_bytes_peak": 113203280896, + "allocated_bytes_current": 72510444544, + "allocated_bytes_peak": 113203280896, + "device": 4, + "inactive_split_bytes_current": 8378805248, + "inactive_split_bytes_peak": 14577149440, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116685537280, + "reserved_bytes_peak": 116685537280, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:44:50.649749Z", + "world_size": 8 + }, + "allocated_bytes_current": 72510444544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113203280896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72510444544, + "active_bytes_peak": 113203280896, + "allocated_bytes_current": 72510444544, + "allocated_bytes_peak": 113203280896, + "device": 4, + "inactive_split_bytes_current": 8672406528, + "inactive_split_bytes_peak": 14187800064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116685537280, + "reserved_bytes_peak": 116685537280, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:44:49.499110Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8378805248, + "inactive_split_bytes_current_delta": -293601280, + "inactive_split_bytes_peak": 14577149440, + "inactive_split_bytes_peak_delta": 389349376, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116685537280, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116685537280, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 468, + "segment_count_current_delta": 0, + "segment_count_peak": 468, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T124450650177Z.pickle", + "timestamp_utc": "2026-08-01T12:44:50.763689Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T124450650177Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T124450650177Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e5031ed76fd142be1ce076e9d8814aa97be0ca57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T124450650177Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a830cb16bf183ba35a24cd9e8d6b837d5ddc1dca67482cd4061ec711433f2d64 +size 5915615 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T124453583750Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T124453583750Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8302c6fab79e8873748794285a50d2efb1a0bc3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T124453583750Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72527488000, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131371147264, + "active_bytes_peak_delta": 18167866368, + "after": { + "active_bytes_current": 72527488000, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 72527488000, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_peak": 21287365632, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148843266048, + "reserved_bytes_peak": 148843266048, + "segment_count_current": 625, + "segment_count_peak": 625, + "timestamp_utc": "2026-08-01T12:44:53.583219Z", + "world_size": 8 + }, + "allocated_bytes_current": 72527488000, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113229446656, + "allocated_bytes_peak_delta": 26165760, + "before": { + "active_bytes_current": 72510444544, + "active_bytes_peak": 113203280896, + "allocated_bytes_current": 72510444544, + "allocated_bytes_peak": 113203280896, + "device": 4, + "inactive_split_bytes_current": 8378805248, + "inactive_split_bytes_peak": 14577149440, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116685537280, + "reserved_bytes_peak": 116685537280, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:44:50.780529Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_current_delta": 9969594368, + "inactive_split_bytes_peak": 21287365632, + "inactive_split_bytes_peak_delta": 6710216192, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148843266048, + "reserved_bytes_current_delta": 32157728768, + "reserved_bytes_peak": 148843266048, + "reserved_bytes_peak_delta": 32157728768, + "segment_count_current": 625, + "segment_count_current_delta": 157, + "segment_count_peak": 625, + "segment_count_peak_delta": 157, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T124453583750Z.pickle", + "timestamp_utc": "2026-08-01T12:44:53.708698Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T124453583750Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T124453583750Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..01c044f245c235b40c535c0043a7a6e3499301a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T124453583750Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3f2df0898a97b52d5540b9fa277ba3487b4b3582950f6923c081d0badcb7fa2 +size 6210038 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T124454826798Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T124454826798Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bcabc9d69a44028044c33dcd820f0dd9dde8a819 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T124454826798Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72527488000, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371147264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72527488000, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 72527488000, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_peak": 21287365632, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148843266048, + "reserved_bytes_peak": 148843266048, + "segment_count_current": 625, + "segment_count_peak": 625, + "timestamp_utc": "2026-08-01T12:44:54.825901Z", + "world_size": 8 + }, + "allocated_bytes_current": 72527488000, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113229446656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72527488000, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 72527488000, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_peak": 21287365632, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148843266048, + "reserved_bytes_peak": 148843266048, + "segment_count_current": 625, + "segment_count_peak": 625, + "timestamp_utc": "2026-08-01T12:44:53.725534Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21287365632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148843266048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148843266048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 625, + "segment_count_current_delta": 0, + "segment_count_peak": 625, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T124454826798Z.pickle", + "timestamp_utc": "2026-08-01T12:44:54.832891Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T124454826798Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T124454826798Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fa929c1bde5fd433debc0379d4439fdc56c7d928 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T124454826798Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:303067a0d1ebd8dc04715d8d0102e7362fd81fa465d443cae759350cc7e742eb +size 171039 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T124455906235Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T124455906235Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6fded3d8b337113020770405c1f70de2ef6ad7ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T124455906235Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72527488000, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371147264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72527488000, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 72527488000, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_peak": 21287365632, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148843266048, + "reserved_bytes_peak": 148843266048, + "segment_count_current": 625, + "segment_count_peak": 625, + "timestamp_utc": "2026-08-01T12:44:55.905779Z", + "world_size": 8 + }, + "allocated_bytes_current": 72527488000, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113229446656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72527488000, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 72527488000, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_peak": 21287365632, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148843266048, + "reserved_bytes_peak": 148843266048, + "segment_count_current": 625, + "segment_count_peak": 625, + "timestamp_utc": "2026-08-01T12:44:54.833835Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21287365632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148843266048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148843266048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 625, + "segment_count_current_delta": 0, + "segment_count_peak": 625, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T124455906235Z.pickle", + "timestamp_utc": "2026-08-01T12:44:55.911177Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T124455906235Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T124455906235Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..daac1d0ce4e1f20ec0398529e71368f7d87c9ea5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T124455906235Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:680d8e4e266ac61148d53643b26a4361f75c5020b396e8276327d6e6c6f30677 +size 171039 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T124456978182Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T124456978182Z.json new file mode 100644 index 0000000000000000000000000000000000000000..723b8ac8fa693d7769a3b1160c55be34f896e5c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T124456978182Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72527488000, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371147264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72527488000, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 72527488000, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_peak": 21287365632, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148843266048, + "reserved_bytes_peak": 148843266048, + "segment_count_current": 625, + "segment_count_peak": 625, + "timestamp_utc": "2026-08-01T12:44:56.977764Z", + "world_size": 8 + }, + "allocated_bytes_current": 72527488000, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113229446656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72527488000, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 72527488000, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_peak": 21287365632, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148843266048, + "reserved_bytes_peak": 148843266048, + "segment_count_current": 625, + "segment_count_peak": 625, + "timestamp_utc": "2026-08-01T12:44:55.912045Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21287365632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148843266048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148843266048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 625, + "segment_count_current_delta": 0, + "segment_count_peak": 625, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T124456978182Z.pickle", + "timestamp_utc": "2026-08-01T12:44:56.983020Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T124456978182Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T124456978182Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..438b87e1b6b4cabd06fbf82e270d303eabcc3bfd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T124456978182Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f8681c10e21b48fd741e2dc9396bac474ab74a2649bdd5e8d31d29b344255cc +size 171039 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T124458046006Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T124458046006Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eb94c9dd03d5e8a3dca6fe90ae10c1b9df9768e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T124458046006Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72527488000, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371147264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72527488000, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 72527488000, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_peak": 21287365632, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148843266048, + "reserved_bytes_peak": 148843266048, + "segment_count_current": 625, + "segment_count_peak": 625, + "timestamp_utc": "2026-08-01T12:44:58.045593Z", + "world_size": 8 + }, + "allocated_bytes_current": 72527488000, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113229446656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72527488000, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 72527488000, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_peak": 21287365632, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148843266048, + "reserved_bytes_peak": 148843266048, + "segment_count_current": 625, + "segment_count_peak": 625, + "timestamp_utc": "2026-08-01T12:44:56.983918Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21287365632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148843266048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148843266048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 625, + "segment_count_current_delta": 0, + "segment_count_peak": 625, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T124458046006Z.pickle", + "timestamp_utc": "2026-08-01T12:44:58.050768Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T124458046006Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T124458046006Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7198d5d4b52ac32cc60d87a769a05774dd9db148 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T124458046006Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be4b83e395d857637ab3792092db3d46a8de25f2a6e6a5cf23c92bff0f4d0d18 +size 171039 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T124459121133Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T124459121133Z.json new file mode 100644 index 0000000000000000000000000000000000000000..133da7ad5f5b9d9b6643491c9998a04e711e9e51 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T124459121133Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72527488000, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371147264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72527488000, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 72527488000, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_peak": 21287365632, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148843266048, + "reserved_bytes_peak": 148843266048, + "segment_count_current": 625, + "segment_count_peak": 625, + "timestamp_utc": "2026-08-01T12:44:59.120719Z", + "world_size": 8 + }, + "allocated_bytes_current": 72527488000, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113229446656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72527488000, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 72527488000, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_peak": 21287365632, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148843266048, + "reserved_bytes_peak": 148843266048, + "segment_count_current": 625, + "segment_count_peak": 625, + "timestamp_utc": "2026-08-01T12:44:58.051673Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21287365632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148843266048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148843266048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 625, + "segment_count_current_delta": 0, + "segment_count_peak": 625, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T124459121133Z.pickle", + "timestamp_utc": "2026-08-01T12:44:59.125969Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T124459121133Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T124459121133Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ac1cb09774fb17107b7f306692402bac85bcb75f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T124459121133Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2da3f6ea2ebb680726118c9c5e552ecf5b29ac30ea7f690bd57260c8c53372c9 +size 171039 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T124558843391Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T124558843391Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7949cdc22a9ba46d48fc51829a1e4afd033fae4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T124558843391Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61320768512, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 131371147264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61320768512, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 61320768512, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 11066627072, + "inactive_split_bytes_peak": 21287365632, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148912472064, + "reserved_bytes_peak": 148912472064, + "segment_count_current": 630, + "segment_count_peak": 630, + "timestamp_utc": "2026-08-01T12:45:58.842898Z", + "world_size": 8 + }, + "allocated_bytes_current": 61320768512, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 113229446656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72527488000, + "active_bytes_peak": 131371147264, + "allocated_bytes_current": 72527488000, + "allocated_bytes_peak": 113229446656, + "device": 4, + "inactive_split_bytes_current": 18348399616, + "inactive_split_bytes_peak": 21287365632, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148843266048, + "reserved_bytes_peak": 148843266048, + "segment_count_current": 625, + "segment_count_peak": 625, + "timestamp_utc": "2026-08-01T12:44:59.126818Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11066627072, + "inactive_split_bytes_current_delta": -7281772544, + "inactive_split_bytes_peak": 21287365632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148912472064, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 148912472064, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 630, + "segment_count_current_delta": 5, + "segment_count_peak": 630, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T124558843391Z.pickle", + "timestamp_utc": "2026-08-01T12:45:58.924936Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T124558843391Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T124558843391Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c14951142db4b4769b25e3df39c1ceadc21ad5d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T124558843391Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed5de98abd68aa58d369597a451fb08c00c56a36e09d13342f7e71dc01f85066 +size 3922891 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T124448034347Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T124448034347Z.json new file mode 100644 index 0000000000000000000000000000000000000000..335fbefa19535ee18d2ef65f8fa4e7b951b109e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T124448034347Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 37277213696, + "active_bytes_peak": 89570930688, + "active_bytes_peak_delta": 54104594432, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 89570930688, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 89570930688, + "device": 5, + "inactive_split_bytes_current": 7971636224, + "inactive_split_bytes_peak": 10777481216, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 98278834176, + "reserved_bytes_peak": 98278834176, + "segment_count_current": 412, + "segment_count_peak": 412, + "timestamp_utc": "2026-08-01T12:44:48.033753Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 37277213696, + "allocated_bytes_peak": 89570930688, + "allocated_bytes_peak_delta": 54104594432, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:43:39.648265Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7971636224, + "inactive_split_bytes_current_delta": 7838817280, + "inactive_split_bytes_peak": 10777481216, + "inactive_split_bytes_peak_delta": 10617095168, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 98278834176, + "reserved_bytes_current_delta": 59034828800, + "reserved_bytes_peak": 98278834176, + "reserved_bytes_peak_delta": 59034828800, + "segment_count_current": 412, + "segment_count_current_delta": 341, + "segment_count_peak": 412, + "segment_count_peak_delta": 341, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T124448034347Z.pickle", + "timestamp_utc": "2026-08-01T12:44:48.151555Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T124448034347Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T124448034347Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1616d176b9c95589a741bde890ca62731b11b9f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T124448034347Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ab2d562cb892ec5b052741de1cd5ff9da46c88dfc9d8763dccedc4e655d55d3 +size 7078221 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T124449364906Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T124449364906Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65ed0dd93a08d5216b4912ec14d6d9e6f37f1af9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T124449364906Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113206119424, + "active_bytes_peak_delta": 23635188736, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113206119424, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113206119424, + "device": 5, + "inactive_split_bytes_current": 9024406528, + "inactive_split_bytes_peak": 13022193152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:49.364493Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113206119424, + "allocated_bytes_peak_delta": 23635188736, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 89570930688, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 89570930688, + "device": 5, + "inactive_split_bytes_current": 7971636224, + "inactive_split_bytes_peak": 10777481216, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 98278834176, + "reserved_bytes_peak": 98278834176, + "segment_count_current": 412, + "segment_count_peak": 412, + "timestamp_utc": "2026-08-01T12:44:48.167308Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9024406528, + "inactive_split_bytes_current_delta": 1052770304, + "inactive_split_bytes_peak": 13022193152, + "inactive_split_bytes_peak_delta": 2244711936, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116821852160, + "reserved_bytes_current_delta": 18543017984, + "reserved_bytes_peak": 116821852160, + "reserved_bytes_peak_delta": 18543017984, + "segment_count_current": 474, + "segment_count_current_delta": 62, + "segment_count_peak": 474, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T124449364906Z.pickle", + "timestamp_utc": "2026-08-01T12:44:49.475380Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T124449364906Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T124449364906Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d5203f84664baf52f8ed4f888f0bdf97546ad7a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T124449364906Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:798edc84e8333ffdfdc325737c537eed2b33103b032f98fb3520858887fe8fec +size 5913385 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T124450650215Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T124450650215Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5d625f6445701525319d7240ac31f5169d902ba4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T124450650215Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113208388608, + "active_bytes_peak_delta": 2269184, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113208388608, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113208388608, + "device": 5, + "inactive_split_bytes_current": 9322202112, + "inactive_split_bytes_peak": 15426136064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116823949312, + "reserved_bytes_peak": 116823949312, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T12:44:50.649809Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113208388608, + "allocated_bytes_peak_delta": 2269184, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113206119424, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113206119424, + "device": 5, + "inactive_split_bytes_current": 9024406528, + "inactive_split_bytes_peak": 13022193152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:49.488380Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9322202112, + "inactive_split_bytes_current_delta": 297795584, + "inactive_split_bytes_peak": 15426136064, + "inactive_split_bytes_peak_delta": 2403942912, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116823949312, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 116823949312, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 475, + "segment_count_current_delta": 1, + "segment_count_peak": 475, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T124450650215Z.pickle", + "timestamp_utc": "2026-08-01T12:44:50.755171Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T124450650215Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T124450650215Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ede2cb659697e61f41a64f5531df75e0c7249530 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T124450650215Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a118a55e056a65b085385f37b2fbf057e6cca9c5f72e411ce6519c3e599eb940 +size 5918426 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T124453583772Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T124453583772Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87f697410faf546be8a592e8bbaf8e7ecce129cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T124453583772Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 18163080192, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:53.583232Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 12728832, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113208388608, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113208388608, + "device": 5, + "inactive_split_bytes_current": 9322202112, + "inactive_split_bytes_peak": 15426136064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116823949312, + "reserved_bytes_peak": 116823949312, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T12:44:50.767263Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 7564161024, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 6249790464, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 32765902848, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 32765902848, + "segment_count_current": 623, + "segment_count_current_delta": 148, + "segment_count_peak": 623, + "segment_count_peak_delta": 148, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T124453583772Z.pickle", + "timestamp_utc": "2026-08-01T12:44:53.705273Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T124453583772Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T124453583772Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b7a97c647e60f25762c0829e5d759a3b5df6cac8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T124453583772Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:602bfbce18a3519d86752f1a69cdc0f92677273a82c1fa96d68beef445018fe8 +size 6202142 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T124454826648Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T124454826648Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bf9005228a26d7a46c38aa21e51f68d2a1ed52f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T124454826648Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:54.825858Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:53.719887Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T124454826648Z.pickle", + "timestamp_utc": "2026-08-01T12:44:54.831062Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T124454826648Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T124454826648Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f9a8a1c54ca20d710b2d6c230fa017b76587c38 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T124454826648Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c9d57b4412434977025b10ba952365167cc0e52157b39905167f7456c2e2609 +size 163461 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T124455906318Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T124455906318Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c9170ae8bab9300d4b50e905b608933761889fb9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T124455906318Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:55.905880Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:54.831955Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T124455906318Z.pickle", + "timestamp_utc": "2026-08-01T12:44:55.912213Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T124455906318Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T124455906318Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e8e3ceac079cde3d4be75ee371b242e508bfe03f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T124455906318Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf8f11b896f0252fe3310d5f35d7dcd5aa81442bda5350d5f1c9af94ee0d4e0e +size 163461 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T124456978165Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T124456978165Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7c351cf356a6aa7e0f6ea901771345ec76cf6e9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T124456978165Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:56.977765Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:55.913085Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T124456978165Z.pickle", + "timestamp_utc": "2026-08-01T12:44:56.982504Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T124456978165Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T124456978165Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b42d754ea487802b913364d65be3ee2dcb2f6744 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T124456978165Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec04bef014924adf1b30fc9666d0753849b76dc335ebd019452b12a12b600c0d +size 163461 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T124458046006Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T124458046006Z.json new file mode 100644 index 0000000000000000000000000000000000000000..44202b8f99453038f1f1a105227d81737b2c17e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T124458046006Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:58.045614Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:56.983407Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T124458046006Z.pickle", + "timestamp_utc": "2026-08-01T12:44:58.050384Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T124458046006Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T124458046006Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2059d11e86f2138662f60db33439cfb563ed5105 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T124458046006Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cc9f5b532736b0b30d6e7517b66b2a20ef03e85f4a2a704b5617cfebef7dfe0 +size 163461 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T124459121166Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T124459121166Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca4461fd794c3760b1d84cfc05dc40529a48e755 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T124459121166Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:59.120744Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:58.051231Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149589852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149589852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 623, + "segment_count_current_delta": 0, + "segment_count_peak": 623, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T124459121166Z.pickle", + "timestamp_utc": "2026-08-01T12:44:59.125722Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T124459121166Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T124459121166Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4cf8c3d8176e80d85cf8b91d24ffe0e72837216 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T124459121166Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4afbd88ad94697ee24cf7cc3dfe152bee5be3cfeb383394e8c14a49f8452f251 +size 163461 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T124558843372Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T124558843372Z.json new file mode 100644 index 0000000000000000000000000000000000000000..138a7f44ea06c6ab6403fcd2a27ad68c451bdc74 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T124558843372Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61318992896, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61318992896, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 61318992896, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 11185843200, + "inactive_split_bytes_peak": 21675926528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150483238912, + "reserved_bytes_peak": 150483238912, + "segment_count_current": 629, + "segment_count_peak": 629, + "timestamp_utc": "2026-08-01T12:45:58.842887Z", + "world_size": 8 + }, + "allocated_bytes_current": 61318992896, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 5, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21675926528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149589852160, + "reserved_bytes_peak": 149589852160, + "segment_count_current": 623, + "segment_count_peak": 623, + "timestamp_utc": "2026-08-01T12:44:59.126579Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11185843200, + "inactive_split_bytes_current_delta": -5700519936, + "inactive_split_bytes_peak": 21675926528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150483238912, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150483238912, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 629, + "segment_count_current_delta": 6, + "segment_count_peak": 629, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T124558843372Z.pickle", + "timestamp_utc": "2026-08-01T12:45:58.923061Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T124558843372Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T124558843372Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..45ad88991843d2457b3c8518da36c10c0c7f8491 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T124558843372Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd9d64ee90224adf34674eafa9d8f80661c4166c6b1c65a7ca3c8672bc062fe7 +size 3841371 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T124448034292Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T124448034292Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2510871f7215c03b3351772b1cd4528a56676615 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T124448034292Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 37277213696, + "active_bytes_peak": 89570930688, + "active_bytes_peak_delta": 54104594432, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 89570930688, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 89570930688, + "device": 6, + "inactive_split_bytes_current": 7971636224, + "inactive_split_bytes_peak": 10777481216, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98278834176, + "reserved_bytes_peak": 98278834176, + "segment_count_current": 412, + "segment_count_peak": 412, + "timestamp_utc": "2026-08-01T12:44:48.033682Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 37277213696, + "allocated_bytes_peak": 89570930688, + "allocated_bytes_peak_delta": 54104594432, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:43:39.647409Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7971636224, + "inactive_split_bytes_current_delta": 7838817280, + "inactive_split_bytes_peak": 10777481216, + "inactive_split_bytes_peak_delta": 10617095168, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98278834176, + "reserved_bytes_current_delta": 59034828800, + "reserved_bytes_peak": 98278834176, + "reserved_bytes_peak_delta": 59034828800, + "segment_count_current": 412, + "segment_count_current_delta": 341, + "segment_count_peak": 412, + "segment_count_peak_delta": 341, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T124448034292Z.pickle", + "timestamp_utc": "2026-08-01T12:44:48.157403Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T124448034292Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T124448034292Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..06babd1ed860742ad9611bd7fa1f15d08d177422 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T124448034292Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e48ff8591721fb49e6441a23274bd6d93d75a84e89211d624f04a4a01fccaba +size 7078258 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T124449365033Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T124449365033Z.json new file mode 100644 index 0000000000000000000000000000000000000000..38aaeed950b81d0fa10a911c43655c7dddc56cbb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T124449365033Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113206119424, + "active_bytes_peak_delta": 23635188736, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113206119424, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113206119424, + "device": 6, + "inactive_split_bytes_current": 9026503680, + "inactive_split_bytes_peak": 13020096000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:49.364538Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113206119424, + "allocated_bytes_peak_delta": 23635188736, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 89570930688, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 89570930688, + "device": 6, + "inactive_split_bytes_current": 7971636224, + "inactive_split_bytes_peak": 10777481216, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98278834176, + "reserved_bytes_peak": 98278834176, + "segment_count_current": 412, + "segment_count_peak": 412, + "timestamp_utc": "2026-08-01T12:44:48.179236Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9026503680, + "inactive_split_bytes_current_delta": 1054867456, + "inactive_split_bytes_peak": 13020096000, + "inactive_split_bytes_peak_delta": 2242614784, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116821852160, + "reserved_bytes_current_delta": 18543017984, + "reserved_bytes_peak": 116821852160, + "reserved_bytes_peak_delta": 18543017984, + "segment_count_current": 474, + "segment_count_current_delta": 62, + "segment_count_peak": 474, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T124449365033Z.pickle", + "timestamp_utc": "2026-08-01T12:44:49.483757Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T124449365033Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T124449365033Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d45acf49b18f8e12707f5ccb781b9a10c7ab1a87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T124449365033Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:599913181922641b2b20314915c1ce1bc16bfd04a5c4d3237155b0874b1640ef +size 5913428 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T124450650338Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T124450650338Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3963886f57bd6dbbf6c1b45f1ca0dfaae8514410 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T124450650338Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508668928, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113208388608, + "active_bytes_peak_delta": 2269184, + "after": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113208388608, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113208388608, + "device": 6, + "inactive_split_bytes_current": 9318007808, + "inactive_split_bytes_peak": 15426136064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:50.649857Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508668928, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113208388608, + "allocated_bytes_peak_delta": 2269184, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113206119424, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113206119424, + "device": 6, + "inactive_split_bytes_current": 9026503680, + "inactive_split_bytes_peak": 13020096000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:49.501403Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9318007808, + "inactive_split_bytes_current_delta": 291504128, + "inactive_split_bytes_peak": 15426136064, + "inactive_split_bytes_peak_delta": 2406040064, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116821852160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116821852160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T124450650338Z.pickle", + "timestamp_utc": "2026-08-01T12:44:50.770046Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T124450650338Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T124450650338Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..88b6b9c3e8c3b589791016556cbb615fc1ad7946 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T124450650338Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0026f84ae2076c839aa481457441fe2f4f2e346eddbcbae62b3910f5bc959432 +size 5918385 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T124453585247Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T124453585247Z.json new file mode 100644 index 0000000000000000000000000000000000000000..89bb42695748d2b00ee6d84e65e9b0903b2ab55e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T124453585247Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 18163080192, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21671732224, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 149585657856, + "reserved_bytes_peak": 149585657856, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-01T12:44:53.584000Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 12728832, + "before": { + "active_bytes_current": 72508668928, + "active_bytes_peak": 113208388608, + "allocated_bytes_current": 72508668928, + "allocated_bytes_peak": 113208388608, + "device": 6, + "inactive_split_bytes_current": 9318007808, + "inactive_split_bytes_peak": 15426136064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116821852160, + "reserved_bytes_peak": 116821852160, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T12:44:50.790344Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 7568355328, + "inactive_split_bytes_peak": 21671732224, + "inactive_split_bytes_peak_delta": 6245596160, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 149585657856, + "reserved_bytes_current_delta": 32763805696, + "reserved_bytes_peak": 149585657856, + "reserved_bytes_peak_delta": 32763805696, + "segment_count_current": 621, + "segment_count_current_delta": 147, + "segment_count_peak": 621, + "segment_count_peak_delta": 147, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T124453585247Z.pickle", + "timestamp_utc": "2026-08-01T12:44:53.728253Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T124453585247Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T124453585247Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a035106ffbfc633799515ee18f31f9a3466048cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T124453585247Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bee528ec31aaf8006fd47f6e37725a4363cbe571904696923ecef4ef30bc350d +size 6201984 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T124454826996Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T124454826996Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87d421692ce8b8a077d88dc17b61161f203f663f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T124454826996Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21671732224, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 149585657856, + "reserved_bytes_peak": 149585657856, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-01T12:44:54.825983Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21671732224, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 149585657856, + "reserved_bytes_peak": 149585657856, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-01T12:44:53.748833Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21671732224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 149585657856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149585657856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 621, + "segment_count_current_delta": 0, + "segment_count_peak": 621, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T124454826996Z.pickle", + "timestamp_utc": "2026-08-01T12:44:54.832085Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T124454826996Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T124454826996Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..75fa7b3d217a64c6eecd3a0cf4929f0814cdb5dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T124454826996Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:008c662c591cffe23bdaf1fae5622973958386ac06829e64f1b1ae0b799610e6 +size 163251 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T124455906317Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T124455906317Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4bb6f52722f28aea630564677b174f0ef19e9fb3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T124455906317Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21671732224, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 149585657856, + "reserved_bytes_peak": 149585657856, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-01T12:44:55.905879Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21671732224, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 149585657856, + "reserved_bytes_peak": 149585657856, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-01T12:44:54.833041Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21671732224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 149585657856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149585657856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 621, + "segment_count_current_delta": 0, + "segment_count_peak": 621, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T124455906317Z.pickle", + "timestamp_utc": "2026-08-01T12:44:55.911129Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T124455906317Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T124455906317Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb5bb1a53adf307752a171723e1d77ebb6591dc4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T124455906317Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:146fb269e826564ed61ab2c14f5c2b9e20b6dc2654634c82f392fd97321b2a50 +size 163251 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T124456978364Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T124456978364Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b480c563f0b401491f225cb883ad70737edacb5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T124456978364Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21671732224, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 149585657856, + "reserved_bytes_peak": 149585657856, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-01T12:44:56.977956Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21671732224, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 149585657856, + "reserved_bytes_peak": 149585657856, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-01T12:44:55.912014Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21671732224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 149585657856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149585657856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 621, + "segment_count_current_delta": 0, + "segment_count_peak": 621, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T124456978364Z.pickle", + "timestamp_utc": "2026-08-01T12:44:56.983696Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T124456978364Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T124456978364Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..943ee0f6df73edaf09c8d954049ef696772afedd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T124456978364Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f30778ada1c9df5f8e0112a5af8ba090e37d907e1e85d367230436c551ed466 +size 163251 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T124458046415Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T124458046415Z.json new file mode 100644 index 0000000000000000000000000000000000000000..045c2a4c6c84507ad68c17111734959a7813bbf2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T124458046415Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21671732224, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 149585657856, + "reserved_bytes_peak": 149585657856, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-01T12:44:58.045962Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21671732224, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 149585657856, + "reserved_bytes_peak": 149585657856, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-01T12:44:56.984579Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21671732224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 149585657856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149585657856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 621, + "segment_count_current_delta": 0, + "segment_count_peak": 621, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T124458046415Z.pickle", + "timestamp_utc": "2026-08-01T12:44:58.053513Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T124458046415Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T124458046415Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..572a446ccbd7c753f70d4539fbb482dca82cf8e7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T124458046415Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ae00b52eaf5e4433476a6c628535386b3a1cb3b63216b5bf1e9c0be6313e244 +size 163251 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T124459121329Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T124459121329Z.json new file mode 100644 index 0000000000000000000000000000000000000000..646bf26ac4336cfcc3ed52b475bac4c67f5e5e49 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T124459121329Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712384, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21671732224, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 149585657856, + "reserved_bytes_peak": 149585657856, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-01T12:44:59.120892Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712384, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21671732224, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 149585657856, + "reserved_bytes_peak": 149585657856, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-01T12:44:58.054463Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21671732224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 149585657856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149585657856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 621, + "segment_count_current_delta": 0, + "segment_count_peak": 621, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T124459121329Z.pickle", + "timestamp_utc": "2026-08-01T12:44:59.126507Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T124459121329Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T124459121329Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..054ae0d81fa7881503713a120caf28a993a5de4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T124459121329Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:181645251d7c79de70c84e3ed5b6d33e65a47bab24eb1f9cd11f6670d878fd96 +size 163251 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T124558843370Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T124558843370Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f599b42173a7443ac692d48dcc4c6b3a71109d6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T124558843370Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61318992896, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 131371468800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61318992896, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 61318992896, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 11185843200, + "inactive_split_bytes_peak": 21671732224, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 150479044608, + "reserved_bytes_peak": 150479044608, + "segment_count_current": 627, + "segment_count_peak": 627, + "timestamp_utc": "2026-08-01T12:45:58.842896Z", + "world_size": 8 + }, + "allocated_bytes_current": 61318992896, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 113221117440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712384, + "active_bytes_peak": 131371468800, + "allocated_bytes_current": 72525712384, + "allocated_bytes_peak": 113221117440, + "device": 6, + "inactive_split_bytes_current": 16886363136, + "inactive_split_bytes_peak": 21671732224, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 149585657856, + "reserved_bytes_peak": 149585657856, + "segment_count_current": 621, + "segment_count_peak": 621, + "timestamp_utc": "2026-08-01T12:44:59.127412Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 11185843200, + "inactive_split_bytes_current_delta": -5700519936, + "inactive_split_bytes_peak": 21671732224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 150479044608, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150479044608, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 627, + "segment_count_current_delta": 6, + "segment_count_peak": 627, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T124558843370Z.pickle", + "timestamp_utc": "2026-08-01T12:45:58.928335Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T124558843370Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T124558843370Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3bc0ba7ff8e50f42fa46b1e8a534d67be489118b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T124558843370Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c63cab2dc211c652ddaac98e7e7866d646a35309ce2234cd472e572d340c217 +size 3841165 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T124448034276Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T124448034276Z.json new file mode 100644 index 0000000000000000000000000000000000000000..57cbd19206090fe457a4554a71e92f2e400ef087 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T124448034276Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508669440, + "active_bytes_current_delta": 37277214208, + "active_bytes_peak": 89570930688, + "active_bytes_peak_delta": 54104594432, + "after": { + "active_bytes_current": 72508669440, + "active_bytes_peak": 89570930688, + "allocated_bytes_current": 72508669440, + "allocated_bytes_peak": 89570930688, + "device": 7, + "inactive_split_bytes_current": 7969538560, + "inactive_split_bytes_peak": 10777481216, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 98278834176, + "reserved_bytes_peak": 98278834176, + "segment_count_current": 412, + "segment_count_peak": 412, + "timestamp_utc": "2026-08-01T12:44:48.033700Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508669440, + "allocated_bytes_current_delta": 37277214208, + "allocated_bytes_peak": 89570930688, + "allocated_bytes_peak_delta": 54104594432, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:43:39.648994Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7969538560, + "inactive_split_bytes_current_delta": 7836719616, + "inactive_split_bytes_peak": 10777481216, + "inactive_split_bytes_peak_delta": 10617095168, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 98278834176, + "reserved_bytes_current_delta": 59034828800, + "reserved_bytes_peak": 98278834176, + "reserved_bytes_peak_delta": 59034828800, + "segment_count_current": 412, + "segment_count_current_delta": 341, + "segment_count_peak": 412, + "segment_count_peak_delta": 341, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T124448034276Z.pickle", + "timestamp_utc": "2026-08-01T12:44:48.154867Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T124448034276Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T124448034276Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..323d5e9d2f6aca6dbdcc86cb2ab70f4dda159029 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T124448034276Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50e2c0d90fc82b20523e7962668268cf2f8d27bbe32150a6a7261c2fad8fe036 +size 7079863 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T124449364852Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T124449364852Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b4cabf2a7e5368e5cf4ee7a990616dff0f3fcdf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T124449364852Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508669440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113206119936, + "active_bytes_peak_delta": 23635189248, + "after": { + "active_bytes_current": 72508669440, + "active_bytes_peak": 113206119936, + "allocated_bytes_current": 72508669440, + "allocated_bytes_peak": 113206119936, + "device": 7, + "inactive_split_bytes_current": 9024406016, + "inactive_split_bytes_peak": 13022192640, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116823949312, + "reserved_bytes_peak": 116823949312, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T12:44:49.364451Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508669440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113206119936, + "allocated_bytes_peak_delta": 23635189248, + "before": { + "active_bytes_current": 72508669440, + "active_bytes_peak": 89570930688, + "allocated_bytes_current": 72508669440, + "allocated_bytes_peak": 89570930688, + "device": 7, + "inactive_split_bytes_current": 7969538560, + "inactive_split_bytes_peak": 10777481216, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 98278834176, + "reserved_bytes_peak": 98278834176, + "segment_count_current": 412, + "segment_count_peak": 412, + "timestamp_utc": "2026-08-01T12:44:48.170334Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9024406016, + "inactive_split_bytes_current_delta": 1054867456, + "inactive_split_bytes_peak": 13022192640, + "inactive_split_bytes_peak_delta": 2244711424, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116823949312, + "reserved_bytes_current_delta": 18545115136, + "reserved_bytes_peak": 116823949312, + "reserved_bytes_peak_delta": 18545115136, + "segment_count_current": 475, + "segment_count_current_delta": 63, + "segment_count_peak": 475, + "segment_count_peak_delta": 63, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T124449364852Z.pickle", + "timestamp_utc": "2026-08-01T12:44:49.469675Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T124449364852Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T124449364852Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fcde35e8d59217171647bc08baa6554693aaee25 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T124449364852Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72835039d8d0ffce79fa9f92d0d7981e78d70db0ae572a8d56213e9309cd1ccb +size 5915103 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T124450650177Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T124450650177Z.json new file mode 100644 index 0000000000000000000000000000000000000000..88d8d810081037f2786a707c262b159e7163f90e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T124450650177Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508669440, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113208389120, + "active_bytes_peak_delta": 2269184, + "after": { + "active_bytes_current": 72508669440, + "active_bytes_peak": 113208389120, + "allocated_bytes_current": 72508669440, + "allocated_bytes_peak": 113208389120, + "device": 7, + "inactive_split_bytes_current": 9324298752, + "inactive_split_bytes_peak": 15426135552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116823949312, + "reserved_bytes_peak": 116823949312, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T12:44:50.649780Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508669440, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113208389120, + "allocated_bytes_peak_delta": 2269184, + "before": { + "active_bytes_current": 72508669440, + "active_bytes_peak": 113206119936, + "allocated_bytes_current": 72508669440, + "allocated_bytes_peak": 113206119936, + "device": 7, + "inactive_split_bytes_current": 9024406016, + "inactive_split_bytes_peak": 13022192640, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116823949312, + "reserved_bytes_peak": 116823949312, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T12:44:49.481746Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9324298752, + "inactive_split_bytes_current_delta": 299892736, + "inactive_split_bytes_peak": 15426135552, + "inactive_split_bytes_peak_delta": 2403942912, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116823949312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116823949312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 475, + "segment_count_current_delta": 0, + "segment_count_peak": 475, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T124450650177Z.pickle", + "timestamp_utc": "2026-08-01T12:44:50.757633Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T124450650177Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T124450650177Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f60e76abca4919382a2ac77be3182fae7608a702 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T124450650177Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b5da1078ee229eafacd2a8e0d0e7181bb68e3f092628aa80c6d841c4888b913 +size 5920072 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T124453583653Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T124453583653Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7311199c26ab894d105c546c6e834814515df387 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T124453583653Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712896, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 131371469312, + "active_bytes_peak_delta": 18163080192, + "after": { + "active_bytes_current": 72525712896, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 72525712896, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_peak": 21678023168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149591949312, + "reserved_bytes_peak": 149591949312, + "segment_count_current": 624, + "segment_count_peak": 624, + "timestamp_utc": "2026-08-01T12:44:53.583163Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712896, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113221117952, + "allocated_bytes_peak_delta": 12728832, + "before": { + "active_bytes_current": 72508669440, + "active_bytes_peak": 113208389120, + "allocated_bytes_current": 72508669440, + "allocated_bytes_peak": 113208389120, + "device": 7, + "inactive_split_bytes_current": 9324298752, + "inactive_split_bytes_peak": 15426135552, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116823949312, + "reserved_bytes_peak": 116823949312, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T12:44:50.771695Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_current_delta": 7562063872, + "inactive_split_bytes_peak": 21678023168, + "inactive_split_bytes_peak_delta": 6251887616, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149591949312, + "reserved_bytes_current_delta": 32768000000, + "reserved_bytes_peak": 149591949312, + "reserved_bytes_peak_delta": 32768000000, + "segment_count_current": 624, + "segment_count_current_delta": 149, + "segment_count_peak": 624, + "segment_count_peak_delta": 149, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T124453583653Z.pickle", + "timestamp_utc": "2026-08-01T12:44:53.702004Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T124453583653Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T124453583653Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..31c48f0741c24b96e2093ba5e99098c2e46024d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T124453583653Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:887e7353fd2867404924aa2d0f8b9a3a5fff8d5fe36836023235504c90100603 +size 6203872 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T124454826765Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T124454826765Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e669a71c7757481f61cf4700c2f023ddeffcbf5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T124454826765Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712896, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371469312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712896, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 72525712896, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_peak": 21678023168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149591949312, + "reserved_bytes_peak": 149591949312, + "segment_count_current": 624, + "segment_count_peak": 624, + "timestamp_utc": "2026-08-01T12:44:54.825837Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712896, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712896, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 72525712896, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_peak": 21678023168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149591949312, + "reserved_bytes_peak": 149591949312, + "segment_count_current": 624, + "segment_count_peak": 624, + "timestamp_utc": "2026-08-01T12:44:53.716006Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21678023168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149591949312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149591949312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 624, + "segment_count_current_delta": 0, + "segment_count_peak": 624, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T124454826765Z.pickle", + "timestamp_utc": "2026-08-01T12:44:54.831260Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T124454826765Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T124454826765Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..afc2840006296f408ed60f6280dac134230fae33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T124454826765Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aafcdda2eb59ea620eadfdae4bca0c5ed8cb619b572a096eb4b2535e6495a08 +size 165274 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T124455906230Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T124455906230Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e65914563f8127760f4d7537007fa9ee8c0e6bc0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T124455906230Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712896, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371469312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712896, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 72525712896, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_peak": 21678023168, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149591949312, + "reserved_bytes_peak": 149591949312, + "segment_count_current": 624, + "segment_count_peak": 624, + "timestamp_utc": "2026-08-01T12:44:55.905818Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712896, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712896, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 72525712896, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_peak": 21678023168, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149591949312, + "reserved_bytes_peak": 149591949312, + "segment_count_current": 624, + "segment_count_peak": 624, + "timestamp_utc": "2026-08-01T12:44:54.832190Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21678023168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149591949312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149591949312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 624, + "segment_count_current_delta": 0, + "segment_count_peak": 624, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T124455906230Z.pickle", + "timestamp_utc": "2026-08-01T12:44:55.910735Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T124455906230Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T124455906230Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e5a8acc1c1cdca29d0dd3abae38e55a5300e98f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T124455906230Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:197eea7b046091fbf9fbf693d8e4ba09b5c2a6826b4557b8e93596294c7924d3 +size 165274 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T124456978215Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T124456978215Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f33d0ea4d0a8b52a8d3d45af161b21744a55350d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T124456978215Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712896, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371469312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712896, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 72525712896, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_peak": 21678023168, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149591949312, + "reserved_bytes_peak": 149591949312, + "segment_count_current": 624, + "segment_count_peak": 624, + "timestamp_utc": "2026-08-01T12:44:56.977803Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712896, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712896, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 72525712896, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_peak": 21678023168, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149591949312, + "reserved_bytes_peak": 149591949312, + "segment_count_current": 624, + "segment_count_peak": 624, + "timestamp_utc": "2026-08-01T12:44:55.911598Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21678023168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149591949312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149591949312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 624, + "segment_count_current_delta": 0, + "segment_count_peak": 624, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T124456978215Z.pickle", + "timestamp_utc": "2026-08-01T12:44:56.982831Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T124456978215Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T124456978215Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5dca615d651166cdd3a8553a99fe3a442c83acc4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T124456978215Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa97d31bcd33fdf855875b416d7b910f4498b3b512c6b3eaa1bc1030ab058a6c +size 165243 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T124458046102Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T124458046102Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4fe0df1ecfd62e9f7c4f7ce71b95f646ee82222b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T124458046102Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712896, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371469312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712896, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 72525712896, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_peak": 21678023168, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149591949312, + "reserved_bytes_peak": 149591949312, + "segment_count_current": 624, + "segment_count_peak": 624, + "timestamp_utc": "2026-08-01T12:44:58.045658Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712896, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712896, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 72525712896, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_peak": 21678023168, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149591949312, + "reserved_bytes_peak": 149591949312, + "segment_count_current": 624, + "segment_count_peak": 624, + "timestamp_utc": "2026-08-01T12:44:56.983689Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21678023168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149591949312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149591949312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 624, + "segment_count_current_delta": 0, + "segment_count_peak": 624, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T124458046102Z.pickle", + "timestamp_utc": "2026-08-01T12:44:58.051042Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T124458046102Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T124458046102Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..702210adc76c59e42dd04f82eac5e06b310d546a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T124458046102Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c27cac2a4b3b2e61b14e961173c3a4ef45a899ab98e8cd11b030c7f369e90381 +size 165274 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T124459121212Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T124459121212Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ead3b70ab9a65a710ddfdb1d515ca6bfea9c47a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T124459121212Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72525712896, + "active_bytes_current_delta": 0, + "active_bytes_peak": 131371469312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72525712896, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 72525712896, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_peak": 21678023168, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149591949312, + "reserved_bytes_peak": 149591949312, + "segment_count_current": 624, + "segment_count_peak": 624, + "timestamp_utc": "2026-08-01T12:44:59.120788Z", + "world_size": 8 + }, + "allocated_bytes_current": 72525712896, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113221117952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712896, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 72525712896, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_peak": 21678023168, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149591949312, + "reserved_bytes_peak": 149591949312, + "segment_count_current": 624, + "segment_count_peak": 624, + "timestamp_utc": "2026-08-01T12:44:58.051915Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21678023168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149591949312, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149591949312, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 624, + "segment_count_current_delta": 0, + "segment_count_peak": 624, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T124459121212Z.pickle", + "timestamp_utc": "2026-08-01T12:44:59.125829Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T124459121212Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T124459121212Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3760820d9063fd36561d400d723f400013d93d54 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T124459121212Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:906dfef79422ee69a7e134c551a194c7c6d9159bf4ea9e169337fca8cefa206e +size 165274 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T124558843362Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T124558843362Z.json new file mode 100644 index 0000000000000000000000000000000000000000..665aad3e7886d6ca962b9b84779a9285d4c5306a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T124558843362Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61318993408, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 131371469312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61318993408, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 61318993408, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 11185842688, + "inactive_split_bytes_peak": 21678023168, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 150485336064, + "reserved_bytes_peak": 150485336064, + "segment_count_current": 630, + "segment_count_peak": 630, + "timestamp_utc": "2026-08-01T12:45:58.842878Z", + "world_size": 8 + }, + "allocated_bytes_current": 61318993408, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 113221117952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72525712896, + "active_bytes_peak": 131371469312, + "allocated_bytes_current": 72525712896, + "allocated_bytes_peak": 113221117952, + "device": 7, + "inactive_split_bytes_current": 16886362624, + "inactive_split_bytes_peak": 21678023168, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149591949312, + "reserved_bytes_peak": 149591949312, + "segment_count_current": 624, + "segment_count_peak": 624, + "timestamp_utc": "2026-08-01T12:44:59.126690Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11185842688, + "inactive_split_bytes_current_delta": -5700519936, + "inactive_split_bytes_peak": 21678023168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 150485336064, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150485336064, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 630, + "segment_count_current_delta": 6, + "segment_count_peak": 630, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T124558843362Z.pickle", + "timestamp_utc": "2026-08-01T12:45:58.924303Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T124558843362Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T124558843362Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f9837e3e23c13a42c95a8f50c49bd413bf5d034 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T124558843362Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06248c39cf4ecb7043b28266a82896825fb781653274615f7581592f7ddfe3aa +size 3843211 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T124852650337Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T124852650337Z.json new file mode 100644 index 0000000000000000000000000000000000000000..86689fdfa4d1afb72ee7e5fbcde260d54c77cfae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T124852650337Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 89669754368, + "active_bytes_peak_delta": 54203418112, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89669754368, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89669754368, + "device": 0, + "inactive_split_bytes_current": 8662336512, + "inactive_split_bytes_peak": 10584691712, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 97913929728, + "reserved_bytes_peak": 97913929728, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.649886Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 89669754368, + "allocated_bytes_peak_delta": 54203418112, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:47:45.021348Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8662336512, + "inactive_split_bytes_current_delta": 8527420416, + "inactive_split_bytes_peak": 10584691712, + "inactive_split_bytes_peak_delta": 10424059904, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 97913929728, + "reserved_bytes_current_delta": 58667827200, + "reserved_bytes_peak": 97913929728, + "reserved_bytes_peak_delta": 58667827200, + "segment_count_current": 468, + "segment_count_current_delta": 396, + "segment_count_peak": 468, + "segment_count_peak_delta": 396, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T124852650337Z.pickle", + "timestamp_utc": "2026-08-01T12:48:52.761068Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T124852650337Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T124852650337Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f072bfda98da14fb6d184250f89cbb9b08df0f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T124852650337Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68ebadd8ec35cb6894eb583c526c5ea77b3a8c0c53c5d755d43fcf82369fccdb +size 5499376 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T124853830858Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T124853830858Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5874f2b090c36ebd7eee2bdcda03ba62f5753c2d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T124853830858Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113306835968, + "active_bytes_peak_delta": 23637081600, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113306835968, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113306835968, + "device": 0, + "inactive_split_bytes_current": 8446329856, + "inactive_split_bytes_peak": 10584691712, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116765229056, + "reserved_bytes_peak": 116765229056, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T12:48:53.830368Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113306835968, + "allocated_bytes_peak_delta": 23637081600, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89669754368, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89669754368, + "device": 0, + "inactive_split_bytes_current": 8662336512, + "inactive_split_bytes_peak": 10584691712, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 97913929728, + "reserved_bytes_peak": 97913929728, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.778112Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8446329856, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 10584691712, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116765229056, + "reserved_bytes_current_delta": 18851299328, + "reserved_bytes_peak": 116765229056, + "reserved_bytes_peak_delta": 18851299328, + "segment_count_current": 530, + "segment_count_current_delta": 62, + "segment_count_peak": 530, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T124853830858Z.pickle", + "timestamp_utc": "2026-08-01T12:48:53.936157Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T124853830858Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T124853830858Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e91d4376825c0bd19f67769e4d7d0c7a4709674 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T124853830858Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6a660ccaf6534c954a693c10b88b18b47d7381988c6014734cd33392cff0474 +size 5579854 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T124854963287Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T124854963287Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c8588a363842f92db1d3a09ecff29c91854aa164 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T124854963287Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113306835968, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113306835968, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113306835968, + "device": 0, + "inactive_split_bytes_current": 8448427008, + "inactive_split_bytes_peak": 11127754240, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117234991104, + "reserved_bytes_peak": 117234991104, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T12:48:54.962755Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113306835968, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113306835968, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113306835968, + "device": 0, + "inactive_split_bytes_current": 8446329856, + "inactive_split_bytes_peak": 10584691712, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116765229056, + "reserved_bytes_peak": 116765229056, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T12:48:53.949709Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8448427008, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11127754240, + "inactive_split_bytes_peak_delta": 543062528, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117234991104, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 117234991104, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 532, + "segment_count_current_delta": 2, + "segment_count_peak": 532, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T124854963287Z.pickle", + "timestamp_utc": "2026-08-01T12:48:55.061477Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T124854963287Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T124854963287Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f586859b92ea758ddef1a5dacd1e5dca85db7b1c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T124854963287Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f142345722924ea2d4d5d8f3e67ee6c83880cc5a1b765a788e9800e0fe875562 +size 5582211 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T124857085795Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T124857085795Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6128026de08484108e78bd9a91cbb496c0261e3d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T124857085795Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 136237534208, + "active_bytes_current_delta": 69316415488, + "active_bytes_peak": 165288667136, + "active_bytes_peak_delta": 51981831168, + "after": { + "active_bytes_current": 136237534208, + "active_bytes_peak": 165288667136, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113323443712, + "device": 0, + "inactive_split_bytes_current": 13597684736, + "inactive_split_bytes_peak": 16054598656, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 182880043008, + "reserved_bytes_peak": 182880043008, + "segment_count_current": 885, + "segment_count_peak": 885, + "timestamp_utc": "2026-08-01T12:48:57.085118Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113323443712, + "allocated_bytes_peak_delta": 16607744, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113306835968, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113306835968, + "device": 0, + "inactive_split_bytes_current": 8448427008, + "inactive_split_bytes_peak": 11127754240, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117234991104, + "reserved_bytes_peak": 117234991104, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T12:48:55.073611Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 13597684736, + "inactive_split_bytes_current_delta": 5149257728, + "inactive_split_bytes_peak": 16054598656, + "inactive_split_bytes_peak_delta": 4926844416, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 182880043008, + "reserved_bytes_current_delta": 65645051904, + "reserved_bytes_peak": 182880043008, + "reserved_bytes_peak_delta": 65645051904, + "segment_count_current": 885, + "segment_count_current_delta": 353, + "segment_count_peak": 885, + "segment_count_peak_delta": 353, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T124857085795Z.pickle", + "timestamp_utc": "2026-08-01T12:48:57.204179Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T124857085795Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T124857085795Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76f211996676ca7f430ffa41092a2811d949737e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T124857085795Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6f6a499b1989a905a2f66faa5579c11b5e15c6bdee5c50c8caa04ecce996fa +size 6563131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T124852650108Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T124852650108Z.json new file mode 100644 index 0000000000000000000000000000000000000000..00024abfd8b7368a56065ee7f25afa411b94954c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T124852650108Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 89674305536, + "active_bytes_peak_delta": 54207969280, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89674305536, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89674305536, + "device": 1, + "inactive_split_bytes_current": 8742028288, + "inactive_split_bytes_peak": 10892579840, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 98322874368, + "reserved_bytes_peak": 98322874368, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.649500Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 89674305536, + "allocated_bytes_peak_delta": 54207969280, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:47:45.022196Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8742028288, + "inactive_split_bytes_current_delta": 8609209344, + "inactive_split_bytes_peak": 10892579840, + "inactive_split_bytes_peak_delta": 10732193792, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 98322874368, + "reserved_bytes_current_delta": 59078868992, + "reserved_bytes_peak": 98322874368, + "reserved_bytes_peak_delta": 59078868992, + "segment_count_current": 468, + "segment_count_current_delta": 397, + "segment_count_peak": 468, + "segment_count_peak_delta": 397, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T124852650108Z.pickle", + "timestamp_utc": "2026-08-01T12:48:52.765128Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T124852650108Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T124852650108Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..42ef367123f2b21df38e6cdebecd056ad17cfd56 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T124852650108Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81a97e3878df9280cd1a2f996946cddb7033a77690d6366cc4dd8620a82270b7 +size 5395529 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T124853830669Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T124853830669Z.json new file mode 100644 index 0000000000000000000000000000000000000000..afcbcb1e74dffacb8059b7cce6d9802879bc49a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T124853830669Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113308949504, + "active_bytes_peak_delta": 23634643968, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113308949504, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113308949504, + "device": 1, + "inactive_split_bytes_current": 8526021632, + "inactive_split_bytes_peak": 10892579840, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117608284160, + "reserved_bytes_peak": 117608284160, + "segment_count_current": 528, + "segment_count_peak": 528, + "timestamp_utc": "2026-08-01T12:48:53.830256Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113308949504, + "allocated_bytes_peak_delta": 23634643968, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89674305536, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89674305536, + "device": 1, + "inactive_split_bytes_current": 8742028288, + "inactive_split_bytes_peak": 10892579840, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 98322874368, + "reserved_bytes_peak": 98322874368, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.783680Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8526021632, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 10892579840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117608284160, + "reserved_bytes_current_delta": 19285409792, + "reserved_bytes_peak": 117608284160, + "reserved_bytes_peak_delta": 19285409792, + "segment_count_current": 528, + "segment_count_current_delta": 60, + "segment_count_peak": 528, + "segment_count_peak_delta": 60, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T124853830669Z.pickle", + "timestamp_utc": "2026-08-01T12:48:53.935212Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T124853830669Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T124853830669Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6acb5f90c244570b7f3e472d943f3744883710ef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T124853830669Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40394373fb7a9eac33dd9410d56519f06abe04519a51d3046e9632ecb1c3a18c +size 5553334 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T124854967037Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T124854967037Z.json new file mode 100644 index 0000000000000000000000000000000000000000..360d0407602bd986e98fbfb6dac5e420e20769c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T124854967037Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113314585600, + "active_bytes_peak_delta": 5636096, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113314585600, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113314585600, + "device": 1, + "inactive_split_bytes_current": 8523924480, + "inactive_split_bytes_peak": 11186434048, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117843165184, + "reserved_bytes_peak": 117843165184, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T12:48:54.966615Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113314585600, + "allocated_bytes_peak_delta": 5636096, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113308949504, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113308949504, + "device": 1, + "inactive_split_bytes_current": 8526021632, + "inactive_split_bytes_peak": 10892579840, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117608284160, + "reserved_bytes_peak": 117608284160, + "segment_count_current": 528, + "segment_count_peak": 528, + "timestamp_utc": "2026-08-01T12:48:53.949563Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8523924480, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11186434048, + "inactive_split_bytes_peak_delta": 293854208, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117843165184, + "reserved_bytes_current_delta": 234881024, + "reserved_bytes_peak": 117843165184, + "reserved_bytes_peak_delta": 234881024, + "segment_count_current": 529, + "segment_count_current_delta": 1, + "segment_count_peak": 529, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T124854967037Z.pickle", + "timestamp_utc": "2026-08-01T12:48:55.078234Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T124854967037Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T124854967037Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6914ce1ba898535f537c77d0aa3de43af222db5e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T124854967037Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa3e5bcb872a9884ee4cba6acdb6e38faf75864690b1e2c94e8800c727415434 +size 5555662 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T124857070941Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T124857070941Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f60b628a2dc7f114731eb98902ad59395756002 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T124857070941Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 136237534208, + "active_bytes_current_delta": 69316415488, + "active_bytes_peak": 165293647872, + "active_bytes_peak_delta": 51979062272, + "after": { + "active_bytes_current": 136237534208, + "active_bytes_peak": 165293647872, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 113319380480, + "device": 1, + "inactive_split_bytes_current": 13098562560, + "inactive_split_bytes_peak": 16594361856, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 182833905664, + "reserved_bytes_peak": 182833905664, + "segment_count_current": 889, + "segment_count_peak": 889, + "timestamp_utc": "2026-08-01T12:48:57.070409Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 113319380480, + "allocated_bytes_peak_delta": 4794880, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113314585600, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113314585600, + "device": 1, + "inactive_split_bytes_current": 8523924480, + "inactive_split_bytes_peak": 11186434048, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117843165184, + "reserved_bytes_peak": 117843165184, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T12:48:55.094711Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 13098562560, + "inactive_split_bytes_current_delta": 4574638080, + "inactive_split_bytes_peak": 16594361856, + "inactive_split_bytes_peak_delta": 5407927808, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 182833905664, + "reserved_bytes_current_delta": 64990740480, + "reserved_bytes_peak": 182833905664, + "reserved_bytes_peak_delta": 64990740480, + "segment_count_current": 889, + "segment_count_current_delta": 360, + "segment_count_peak": 889, + "segment_count_peak_delta": 360, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T124857070941Z.pickle", + "timestamp_utc": "2026-08-01T12:48:57.181950Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T124857070941Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T124857070941Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4eccc3ff582a5f144a13248db1dbd15c0fbc38e7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T124857070941Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24383c1b06fbbc41525b176fff5e06cc2b21a4b9d5bc44e21cd7841e2af502f6 +size 6548371 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T124852650093Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T124852650093Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c99f922f301b3c16a84c2c94754f7c3916d1b96e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T124852650093Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 89676402688, + "active_bytes_peak_delta": 54210066432, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89676402688, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89676402688, + "device": 2, + "inactive_split_bytes_current": 8744125440, + "inactive_split_bytes_peak": 10718909440, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98322874368, + "reserved_bytes_peak": 98322874368, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.649496Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 89676402688, + "allocated_bytes_peak_delta": 54210066432, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:47:45.020362Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8744125440, + "inactive_split_bytes_current_delta": 8611306496, + "inactive_split_bytes_peak": 10718909440, + "inactive_split_bytes_peak_delta": 10558523392, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98322874368, + "reserved_bytes_current_delta": 59078868992, + "reserved_bytes_peak": 98322874368, + "reserved_bytes_peak_delta": 59078868992, + "segment_count_current": 468, + "segment_count_current_delta": 397, + "segment_count_peak": 468, + "segment_count_peak_delta": 397, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T124852650093Z.pickle", + "timestamp_utc": "2026-08-01T12:48:52.763042Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T124852650093Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T124852650093Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cd1c445668b86263c48c301c26952fe3e2e34b20 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T124852650093Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e06eaa192c66f5fbabd947beb4893147668e862c3c254dc93d082d99c41132a +size 5395526 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T124853830687Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T124853830687Z.json new file mode 100644 index 0000000000000000000000000000000000000000..74432d0a617d56bd5362094a5e66930b4f02e282 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T124853830687Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113312377856, + "active_bytes_peak_delta": 23635975168, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312377856, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312377856, + "device": 2, + "inactive_split_bytes_current": 8523924480, + "inactive_split_bytes_peak": 10718909440, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117432123392, + "reserved_bytes_peak": 117432123392, + "segment_count_current": 528, + "segment_count_peak": 528, + "timestamp_utc": "2026-08-01T12:48:53.830279Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113312377856, + "allocated_bytes_peak_delta": 23635975168, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89676402688, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89676402688, + "device": 2, + "inactive_split_bytes_current": 8744125440, + "inactive_split_bytes_peak": 10718909440, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98322874368, + "reserved_bytes_peak": 98322874368, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.780732Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8523924480, + "inactive_split_bytes_current_delta": -220200960, + "inactive_split_bytes_peak": 10718909440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117432123392, + "reserved_bytes_current_delta": 19109249024, + "reserved_bytes_peak": 117432123392, + "reserved_bytes_peak_delta": 19109249024, + "segment_count_current": 528, + "segment_count_current_delta": 60, + "segment_count_peak": 528, + "segment_count_peak_delta": 60, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T124853830687Z.pickle", + "timestamp_utc": "2026-08-01T12:48:53.935462Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T124853830687Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T124853830687Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f5908594c4f683ac6d5cb1ade3008594160b662 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T124853830687Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0bab77df896b520aad21aeca90a261c17587a1b1d23edafbc7ea815e706c256 +size 5546323 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T124854963052Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T124854963052Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eaf109111b83b141b942ce3301c42ac5f9fc8599 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T124854963052Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113312377856, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312377856, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312377856, + "device": 2, + "inactive_split_bytes_current": 8521827328, + "inactive_split_bytes_peak": 11087239168, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118136766464, + "reserved_bytes_peak": 118136766464, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T12:48:54.962654Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113312377856, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312377856, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312377856, + "device": 2, + "inactive_split_bytes_current": 8523924480, + "inactive_split_bytes_peak": 10718909440, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117432123392, + "reserved_bytes_peak": 117432123392, + "segment_count_current": 528, + "segment_count_peak": 528, + "timestamp_utc": "2026-08-01T12:48:53.948893Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8521827328, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11087239168, + "inactive_split_bytes_peak_delta": 368329728, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118136766464, + "reserved_bytes_current_delta": 704643072, + "reserved_bytes_peak": 118136766464, + "reserved_bytes_peak_delta": 704643072, + "segment_count_current": 531, + "segment_count_current_delta": 3, + "segment_count_peak": 531, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T124854963052Z.pickle", + "timestamp_utc": "2026-08-01T12:48:55.063630Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T124854963052Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T124854963052Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..429dab81abe5be2e41f5116ec5943a0d248f1b71 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T124854963052Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3b493ec7d60a4f10c00aa7dc4fd5f8efcaf0b263bced662f700206ceed21fcb +size 5550690 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T124856963100Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T124856963100Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b634aa2cb411bdd95d5a5d60fbb5bc3e7c561f64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T124856963100Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 147721004032, + "active_bytes_current_delta": 80799885312, + "active_bytes_peak": 165293910016, + "active_bytes_peak_delta": 51981532160, + "after": { + "active_bytes_current": 147721004032, + "active_bytes_peak": 165293910016, + "allocated_bytes_current": 79303566336, + "allocated_bytes_peak": 113320822272, + "device": 2, + "inactive_split_bytes_current": 15301106688, + "inactive_split_bytes_peak": 15965688832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 182743728128, + "reserved_bytes_peak": 182743728128, + "segment_count_current": 888, + "segment_count_peak": 888, + "timestamp_utc": "2026-08-01T12:48:56.962578Z", + "world_size": 8 + }, + "allocated_bytes_current": 79303566336, + "allocated_bytes_current_delta": 12382447616, + "allocated_bytes_peak": 113320822272, + "allocated_bytes_peak_delta": 8444416, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312377856, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312377856, + "device": 2, + "inactive_split_bytes_current": 8521827328, + "inactive_split_bytes_peak": 11087239168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118136766464, + "reserved_bytes_peak": 118136766464, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T12:48:55.076498Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 15301106688, + "inactive_split_bytes_current_delta": 6779279360, + "inactive_split_bytes_peak": 15965688832, + "inactive_split_bytes_peak_delta": 4878449664, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 182743728128, + "reserved_bytes_current_delta": 64606961664, + "reserved_bytes_peak": 182743728128, + "reserved_bytes_peak_delta": 64606961664, + "segment_count_current": 888, + "segment_count_current_delta": 357, + "segment_count_peak": 888, + "segment_count_peak_delta": 357, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T124856963100Z.pickle", + "timestamp_utc": "2026-08-01T12:48:57.082608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T124856963100Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T124856963100Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a255352d7bb9b2b0caf937f168859c1249aa7ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T124856963100Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a10cfc7f2c6d3a10b04433756d563e026a59297f827510aaf60836a489e81cb1 +size 6500699 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T124852650110Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T124852650110Z.json new file mode 100644 index 0000000000000000000000000000000000000000..165167b01c7ecf54ea3a5cc624960a4f651756a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T124852650110Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 89678499840, + "active_bytes_peak_delta": 54212163584, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89678499840, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89678499840, + "device": 3, + "inactive_split_bytes_current": 8859468800, + "inactive_split_bytes_peak": 11146728448, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 97970552832, + "reserved_bytes_peak": 97970552832, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T12:48:52.649503Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 89678499840, + "allocated_bytes_peak_delta": 54212163584, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:47:45.021100Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8859468800, + "inactive_split_bytes_current_delta": 8726649856, + "inactive_split_bytes_peak": 11146728448, + "inactive_split_bytes_peak_delta": 10986342400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 97970552832, + "reserved_bytes_current_delta": 58726547456, + "reserved_bytes_peak": 97970552832, + "reserved_bytes_peak_delta": 58726547456, + "segment_count_current": 466, + "segment_count_current_delta": 395, + "segment_count_peak": 466, + "segment_count_peak_delta": 395, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T124852650110Z.pickle", + "timestamp_utc": "2026-08-01T12:48:52.764007Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T124852650110Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T124852650110Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f95b033ab163f5eeba7e712ddc06cd276f944d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T124852650110Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3ad236bb6636332961c094274c20669c8dde4f77fe031f7cad7aa8f47df3c97 +size 5395223 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T124853830669Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T124853830669Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d7dcd36dbe2bb884d7200c953476ad9797adabb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T124853830669Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113313537024, + "active_bytes_peak_delta": 23635037184, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113313537024, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113313537024, + "device": 3, + "inactive_split_bytes_current": 8878343168, + "inactive_split_bytes_peak": 11146728448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117285322752, + "reserved_bytes_peak": 117285322752, + "segment_count_current": 527, + "segment_count_peak": 527, + "timestamp_utc": "2026-08-01T12:48:53.830265Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113313537024, + "allocated_bytes_peak_delta": 23635037184, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89678499840, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89678499840, + "device": 3, + "inactive_split_bytes_current": 8859468800, + "inactive_split_bytes_peak": 11146728448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 97970552832, + "reserved_bytes_peak": 97970552832, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T12:48:52.783639Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8878343168, + "inactive_split_bytes_current_delta": 18874368, + "inactive_split_bytes_peak": 11146728448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117285322752, + "reserved_bytes_current_delta": 19314769920, + "reserved_bytes_peak": 117285322752, + "reserved_bytes_peak_delta": 19314769920, + "segment_count_current": 527, + "segment_count_current_delta": 61, + "segment_count_peak": 527, + "segment_count_peak_delta": 61, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T124853830669Z.pickle", + "timestamp_utc": "2026-08-01T12:48:53.936954Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T124853830669Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T124853830669Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2c7427f1ca34c4e1e1b677fdecad590d9f74e873 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T124853830669Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:385acb38c2c6867cf147edb760df8ce33c14176095a2a6ffb3dc72fad5d1ca3b +size 5555377 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T124854963040Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T124854963040Z.json new file mode 100644 index 0000000000000000000000000000000000000000..343123f23e3f8652bd94e374a9bbd0e0fb4f35ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T124854963040Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113313537024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113313537024, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113313537024, + "device": 3, + "inactive_split_bytes_current": 8639267840, + "inactive_split_bytes_peak": 12065751040, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117520203776, + "reserved_bytes_peak": 117520203776, + "segment_count_current": 528, + "segment_count_peak": 528, + "timestamp_utc": "2026-08-01T12:48:54.962650Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113313537024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113313537024, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113313537024, + "device": 3, + "inactive_split_bytes_current": 8878343168, + "inactive_split_bytes_peak": 11146728448, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117285322752, + "reserved_bytes_peak": 117285322752, + "segment_count_current": 527, + "segment_count_peak": 527, + "timestamp_utc": "2026-08-01T12:48:53.952191Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8639267840, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 12065751040, + "inactive_split_bytes_peak_delta": 919022592, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117520203776, + "reserved_bytes_current_delta": 234881024, + "reserved_bytes_peak": 117520203776, + "reserved_bytes_peak_delta": 234881024, + "segment_count_current": 528, + "segment_count_current_delta": 1, + "segment_count_peak": 528, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T124854963040Z.pickle", + "timestamp_utc": "2026-08-01T12:48:55.065984Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T124854963040Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T124854963040Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..30e4a8919ea14d1f78d7fcca6a4afce9a0d2df9a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T124854963040Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e674047e2fc7b496ce0234a5cdab6a22ce844cfa374a46d6af7bc306d8f0b6 +size 5555811 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T124857013098Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T124857013098Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b30b3cb88f95857982314b3503eab8c83aa5c739 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T124857013098Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 147722445824, + "active_bytes_current_delta": 80801327104, + "active_bytes_peak": 165293123584, + "active_bytes_peak_delta": 51979586560, + "after": { + "active_bytes_current": 147722445824, + "active_bytes_peak": 165293123584, + "allocated_bytes_current": 79305008128, + "allocated_bytes_peak": 113319380480, + "device": 3, + "inactive_split_bytes_current": 16436321280, + "inactive_split_bytes_peak": 16661435904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 182743728128, + "reserved_bytes_peak": 182743728128, + "segment_count_current": 886, + "segment_count_peak": 886, + "timestamp_utc": "2026-08-01T12:48:57.012562Z", + "world_size": 8 + }, + "allocated_bytes_current": 79305008128, + "allocated_bytes_current_delta": 12383889408, + "allocated_bytes_peak": 113319380480, + "allocated_bytes_peak_delta": 5843456, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113313537024, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113313537024, + "device": 3, + "inactive_split_bytes_current": 8639267840, + "inactive_split_bytes_peak": 12065751040, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117520203776, + "reserved_bytes_peak": 117520203776, + "segment_count_current": 528, + "segment_count_peak": 528, + "timestamp_utc": "2026-08-01T12:48:55.080740Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16436321280, + "inactive_split_bytes_current_delta": 7797053440, + "inactive_split_bytes_peak": 16661435904, + "inactive_split_bytes_peak_delta": 4595684864, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 182743728128, + "reserved_bytes_current_delta": 65223524352, + "reserved_bytes_peak": 182743728128, + "reserved_bytes_peak_delta": 65223524352, + "segment_count_current": 886, + "segment_count_current_delta": 358, + "segment_count_peak": 886, + "segment_count_peak_delta": 358, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T124857013098Z.pickle", + "timestamp_utc": "2026-08-01T12:48:57.134555Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T124857013098Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T124857013098Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..986f4c2a84885abf35988d35abb54d0ef5bb27fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T124857013098Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f7b1fb528c79842d27af065d5de72842fa73ef1fd2d697878dc60709786d902 +size 6499792 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T124852650048Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T124852650048Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2467237fdb997a74a30071d605ac60eda0c835b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T124852650048Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922890240, + "active_bytes_current_delta": 31690992640, + "active_bytes_peak": 89673555456, + "active_bytes_peak_delta": 54206776832, + "after": { + "active_bytes_current": 66922890240, + "active_bytes_peak": 89673555456, + "allocated_bytes_current": 66922890240, + "allocated_bytes_peak": 89673555456, + "device": 4, + "inactive_split_bytes_current": 8882863104, + "inactive_split_bytes_peak": 10804933632, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 98322874368, + "reserved_bytes_peak": 98322874368, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.649458Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922890240, + "allocated_bytes_current_delta": 31690992640, + "allocated_bytes_peak": 89673555456, + "allocated_bytes_peak_delta": 54206776832, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:47:45.020953Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8882863104, + "inactive_split_bytes_current_delta": 8750486528, + "inactive_split_bytes_peak": 10804933632, + "inactive_split_bytes_peak_delta": 10651524096, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 98322874368, + "reserved_bytes_current_delta": 59078868992, + "reserved_bytes_peak": 98322874368, + "reserved_bytes_peak_delta": 59078868992, + "segment_count_current": 468, + "segment_count_current_delta": 397, + "segment_count_peak": 468, + "segment_count_peak_delta": 397, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T124852650048Z.pickle", + "timestamp_utc": "2026-08-01T12:48:52.761610Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T124852650048Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T124852650048Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ee7d8147c84acf60a99bddb749ba044db03cde16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T124852650048Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8068447fb1a56644c60f07a56f3e454d376968112d4af4d46b0a90dbf6a7aba5 +size 5484641 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T124853830646Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T124853830646Z.json new file mode 100644 index 0000000000000000000000000000000000000000..19cc54112d798e5778220e4de6b33f903b4eef56 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T124853830646Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922890240, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113311702528, + "active_bytes_peak_delta": 23638147072, + "after": { + "active_bytes_current": 66922890240, + "active_bytes_peak": 113311702528, + "allocated_bytes_current": 66922890240, + "allocated_bytes_peak": 113311702528, + "device": 4, + "inactive_split_bytes_current": 8666856448, + "inactive_split_bytes_peak": 10804933632, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117228699648, + "reserved_bytes_peak": 117228699648, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T12:48:53.830232Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922890240, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113311702528, + "allocated_bytes_peak_delta": 23638147072, + "before": { + "active_bytes_current": 66922890240, + "active_bytes_peak": 89673555456, + "allocated_bytes_current": 66922890240, + "allocated_bytes_peak": 89673555456, + "device": 4, + "inactive_split_bytes_current": 8882863104, + "inactive_split_bytes_peak": 10804933632, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 98322874368, + "reserved_bytes_peak": 98322874368, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.779315Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8666856448, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 10804933632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117228699648, + "reserved_bytes_current_delta": 18905825280, + "reserved_bytes_peak": 117228699648, + "reserved_bytes_peak_delta": 18905825280, + "segment_count_current": 529, + "segment_count_current_delta": 61, + "segment_count_peak": 529, + "segment_count_peak_delta": 61, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T124853830646Z.pickle", + "timestamp_utc": "2026-08-01T12:48:53.935041Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T124853830646Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T124853830646Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d8d9dfd37477e2b72cc06e7a84cc3a6401d785b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T124853830646Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d8da0b7fe3c9439dc342ade421d234bc42164fef7363f11a6756638c8addd0c +size 5571930 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T124854963009Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T124854963009Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cdfe57f7027aafaab277d2232fa629c257a739d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T124854963009Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922890240, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113312291840, + "active_bytes_peak_delta": 589312, + "after": { + "active_bytes_current": 66922890240, + "active_bytes_peak": 113312291840, + "allocated_bytes_current": 66922890240, + "allocated_bytes_peak": 113312291840, + "device": 4, + "inactive_split_bytes_current": 8664759296, + "inactive_split_bytes_peak": 11535577088, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117933342720, + "reserved_bytes_peak": 117933342720, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T12:48:54.962601Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922890240, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113312291840, + "allocated_bytes_peak_delta": 589312, + "before": { + "active_bytes_current": 66922890240, + "active_bytes_peak": 113311702528, + "allocated_bytes_current": 66922890240, + "allocated_bytes_peak": 113311702528, + "device": 4, + "inactive_split_bytes_current": 8666856448, + "inactive_split_bytes_peak": 10804933632, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117228699648, + "reserved_bytes_peak": 117228699648, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T12:48:53.948521Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8664759296, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11535577088, + "inactive_split_bytes_peak_delta": 730643456, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117933342720, + "reserved_bytes_current_delta": 704643072, + "reserved_bytes_peak": 117933342720, + "reserved_bytes_peak_delta": 704643072, + "segment_count_current": 532, + "segment_count_current_delta": 3, + "segment_count_peak": 532, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T124854963009Z.pickle", + "timestamp_utc": "2026-08-01T12:48:55.063239Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T124854963009Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T124854963009Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ec44f3d05857704c25bee69a32e0f0eb37b595fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T124854963009Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:515b42b5cbefd72a380152ed4c8bad81c9c6f74dafb159faf217f2da64f9fd36 +size 5567553 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T124856874972Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T124856874972Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e952239affee3e550a1961636cd1a012499508a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T124856874972Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 147725528064, + "active_bytes_current_delta": 80802637824, + "active_bytes_peak": 165297385472, + "active_bytes_peak_delta": 51985093632, + "after": { + "active_bytes_current": 147725528064, + "active_bytes_peak": 165297385472, + "allocated_bytes_current": 79308090368, + "allocated_bytes_peak": 113322856448, + "device": 4, + "inactive_split_bytes_current": 15640515584, + "inactive_split_bytes_peak": 16259615744, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 182888431616, + "reserved_bytes_peak": 182888431616, + "segment_count_current": 890, + "segment_count_peak": 890, + "timestamp_utc": "2026-08-01T12:48:56.874484Z", + "world_size": 8 + }, + "allocated_bytes_current": 79308090368, + "allocated_bytes_current_delta": 12385200128, + "allocated_bytes_peak": 113322856448, + "allocated_bytes_peak_delta": 10564608, + "before": { + "active_bytes_current": 66922890240, + "active_bytes_peak": 113312291840, + "allocated_bytes_current": 66922890240, + "allocated_bytes_peak": 113312291840, + "device": 4, + "inactive_split_bytes_current": 8664759296, + "inactive_split_bytes_peak": 11535577088, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117933342720, + "reserved_bytes_peak": 117933342720, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T12:48:55.076271Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 15640515584, + "inactive_split_bytes_current_delta": 6975756288, + "inactive_split_bytes_peak": 16259615744, + "inactive_split_bytes_peak_delta": 4724038656, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 182888431616, + "reserved_bytes_current_delta": 64955088896, + "reserved_bytes_peak": 182888431616, + "reserved_bytes_peak_delta": 64955088896, + "segment_count_current": 890, + "segment_count_current_delta": 358, + "segment_count_peak": 890, + "segment_count_peak_delta": 358, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T124856874972Z.pickle", + "timestamp_utc": "2026-08-01T12:48:56.992704Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T124856874972Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T124856874972Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0dea9a2b918face0cd463aceedd7d4bcff297454 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T124856874972Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eca5b6be684c2414caa3479f9e07465160065b0ea60a63a95ce29eb526c77382 +size 6232767 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T124852650123Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T124852650123Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e7e5b218f3c66f7b8bdc3d982570f1f84ddb0796 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T124852650123Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 89681514496, + "active_bytes_peak_delta": 54215178240, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89681514496, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89681514496, + "device": 5, + "inactive_split_bytes_current": 9092252672, + "inactive_split_bytes_peak": 11006366720, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 98440314880, + "reserved_bytes_peak": 98440314880, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.649511Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 89681514496, + "allocated_bytes_peak_delta": 54215178240, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:47:45.021316Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9092252672, + "inactive_split_bytes_current_delta": 8959433728, + "inactive_split_bytes_peak": 11006366720, + "inactive_split_bytes_peak_delta": 10845980672, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 98440314880, + "reserved_bytes_current_delta": 59196309504, + "reserved_bytes_peak": 98440314880, + "reserved_bytes_peak_delta": 59196309504, + "segment_count_current": 468, + "segment_count_current_delta": 397, + "segment_count_peak": 468, + "segment_count_peak_delta": 397, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T124852650123Z.pickle", + "timestamp_utc": "2026-08-01T12:48:52.768404Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T124852650123Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T124852650123Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e78382000068f8ebc729b2bbe2a9de51377c89c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T124852650123Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8739261fc341cc06a6e6207db11fa01479ffd9af666e58e39a1e8bfb4dfe1d3 +size 5395608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T124853830648Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T124853830648Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5206481316cac41f1bb84e9ca7f73939137f64e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T124853830648Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113308294144, + "active_bytes_peak_delta": 23626779648, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113308294144, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113308294144, + "device": 5, + "inactive_split_bytes_current": 8876246016, + "inactive_split_bytes_peak": 11006366720, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117725724672, + "reserved_bytes_peak": 117725724672, + "segment_count_current": 528, + "segment_count_peak": 528, + "timestamp_utc": "2026-08-01T12:48:53.830246Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113308294144, + "allocated_bytes_peak_delta": 23626779648, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89681514496, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89681514496, + "device": 5, + "inactive_split_bytes_current": 9092252672, + "inactive_split_bytes_peak": 11006366720, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 98440314880, + "reserved_bytes_peak": 98440314880, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.786688Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8876246016, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 11006366720, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117725724672, + "reserved_bytes_current_delta": 19285409792, + "reserved_bytes_peak": 117725724672, + "reserved_bytes_peak_delta": 19285409792, + "segment_count_current": 528, + "segment_count_current_delta": 60, + "segment_count_peak": 528, + "segment_count_peak_delta": 60, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T124853830648Z.pickle", + "timestamp_utc": "2026-08-01T12:48:53.935951Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T124853830648Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T124853830648Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..49b4224ef7f79bf13a3d70b0bcbf8655ed26dda2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T124853830648Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5183b432298ea4cd1f6bad73c5ccd295c5914903f9fdbd768a54942aa6e88e98 +size 5556548 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T124854963009Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T124854963009Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ac09e7a5ed543d9ef9df24e3522d0a5bf5dc590 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T124854963009Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113312094720, + "active_bytes_peak_delta": 3800576, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312094720, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312094720, + "device": 5, + "inactive_split_bytes_current": 8641364992, + "inactive_split_bytes_peak": 11673755648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118195486720, + "reserved_bytes_peak": 118195486720, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T12:48:54.962609Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113312094720, + "allocated_bytes_peak_delta": 3800576, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113308294144, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113308294144, + "device": 5, + "inactive_split_bytes_current": 8876246016, + "inactive_split_bytes_peak": 11006366720, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117725724672, + "reserved_bytes_peak": 117725724672, + "segment_count_current": 528, + "segment_count_peak": 528, + "timestamp_utc": "2026-08-01T12:48:53.950681Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8641364992, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 11673755648, + "inactive_split_bytes_peak_delta": 667388928, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118195486720, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 118195486720, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 530, + "segment_count_current_delta": 2, + "segment_count_peak": 530, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T124854963009Z.pickle", + "timestamp_utc": "2026-08-01T12:48:55.067838Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T124854963009Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T124854963009Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..be2cd450f40657fbff2ebdf6bda1408e0f35af23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T124854963009Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaaa4a536d3fcd760c0b7d7a22eca2f2af93496450913f75477dc5dc1145f9c2 +size 5556070 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T124856990545Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T124856990545Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b96c24e60a551121dcc471adba28c92d49919b79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T124856990545Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 147721790464, + "active_bytes_current_delta": 80800671744, + "active_bytes_peak": 165294565376, + "active_bytes_peak_delta": 51982470656, + "after": { + "active_bytes_current": 147721790464, + "active_bytes_peak": 165294565376, + "allocated_bytes_current": 79304352768, + "allocated_bytes_peak": 113320036352, + "device": 5, + "inactive_split_bytes_current": 15858162688, + "inactive_split_bytes_peak": 16561427968, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 182842294272, + "reserved_bytes_peak": 182842294272, + "segment_count_current": 879, + "segment_count_peak": 879, + "timestamp_utc": "2026-08-01T12:48:56.990004Z", + "world_size": 8 + }, + "allocated_bytes_current": 79304352768, + "allocated_bytes_current_delta": 12383234048, + "allocated_bytes_peak": 113320036352, + "allocated_bytes_peak_delta": 7941632, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312094720, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312094720, + "device": 5, + "inactive_split_bytes_current": 8641364992, + "inactive_split_bytes_peak": 11673755648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118195486720, + "reserved_bytes_peak": 118195486720, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T12:48:55.082350Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 15858162688, + "inactive_split_bytes_current_delta": 7216797696, + "inactive_split_bytes_peak": 16561427968, + "inactive_split_bytes_peak_delta": 4887672320, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 182842294272, + "reserved_bytes_current_delta": 64646807552, + "reserved_bytes_peak": 182842294272, + "reserved_bytes_peak_delta": 64646807552, + "segment_count_current": 879, + "segment_count_current_delta": 349, + "segment_count_peak": 879, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T124856990545Z.pickle", + "timestamp_utc": "2026-08-01T12:48:57.111622Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T124856990545Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T124856990545Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dcbba096d29639399d2471cee175574043392079 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T124856990545Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3f29d956fb5551cba47b50e394aa83c59ae9a1184366abaefe1c3055711b768 +size 6498705 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T124852650096Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T124852650096Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d309ca607b8486bb392ca2bca86d2696f527223 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T124852650096Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 89681514496, + "active_bytes_peak_delta": 54215178240, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89681514496, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89681514496, + "device": 6, + "inactive_split_bytes_current": 9094349824, + "inactive_split_bytes_peak": 10712765440, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98440314880, + "reserved_bytes_peak": 98440314880, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.649477Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 89681514496, + "allocated_bytes_peak_delta": 54215178240, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:47:45.021195Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9094349824, + "inactive_split_bytes_current_delta": 8961530880, + "inactive_split_bytes_peak": 10712765440, + "inactive_split_bytes_peak_delta": 10552379392, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98440314880, + "reserved_bytes_current_delta": 59196309504, + "reserved_bytes_peak": 98440314880, + "reserved_bytes_peak_delta": 59196309504, + "segment_count_current": 468, + "segment_count_current_delta": 397, + "segment_count_peak": 468, + "segment_count_peak_delta": 397, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T124852650096Z.pickle", + "timestamp_utc": "2026-08-01T12:48:52.760684Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T124852650096Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T124852650096Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3221280f41cf05c791c2789ed9d9420a03145ff9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T124852650096Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:214f0021cac08ed8285cac2a48b4c2fd385b3ae38ea77699df71d98644713e49 +size 5395604 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T124853830661Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T124853830661Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5059f0b360a6d4bc01104556a851f7533e97908a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T124853830661Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113308294144, + "active_bytes_peak_delta": 23626779648, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113308294144, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113308294144, + "device": 6, + "inactive_split_bytes_current": 8641364992, + "inactive_split_bytes_peak": 10712765440, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117490843648, + "reserved_bytes_peak": 117490843648, + "segment_count_current": 527, + "segment_count_peak": 527, + "timestamp_utc": "2026-08-01T12:48:53.830248Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113308294144, + "allocated_bytes_peak_delta": 23626779648, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 89681514496, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 89681514496, + "device": 6, + "inactive_split_bytes_current": 9094349824, + "inactive_split_bytes_peak": 10712765440, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98440314880, + "reserved_bytes_peak": 98440314880, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.779997Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8641364992, + "inactive_split_bytes_current_delta": -452984832, + "inactive_split_bytes_peak": 10712765440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117490843648, + "reserved_bytes_current_delta": 19050528768, + "reserved_bytes_peak": 117490843648, + "reserved_bytes_peak_delta": 19050528768, + "segment_count_current": 527, + "segment_count_current_delta": 59, + "segment_count_peak": 527, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T124853830661Z.pickle", + "timestamp_utc": "2026-08-01T12:48:53.936482Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T124853830661Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T124853830661Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf9813e12682bce726b001750bbc237408aaa9ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T124853830661Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:983927070a7a4306920688180dc27d718f915df418a88b43a3de259300e7b89a +size 5553907 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T124854963026Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T124854963026Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e45f4426a21984cb7855defd118746e15cf9819a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T124854963026Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113312094720, + "active_bytes_peak_delta": 3800576, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312094720, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312094720, + "device": 6, + "inactive_split_bytes_current": 8639267840, + "inactive_split_bytes_peak": 11675852800, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118195486720, + "reserved_bytes_peak": 118195486720, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T12:48:54.962622Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113312094720, + "allocated_bytes_peak_delta": 3800576, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113308294144, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113308294144, + "device": 6, + "inactive_split_bytes_current": 8641364992, + "inactive_split_bytes_peak": 10712765440, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117490843648, + "reserved_bytes_peak": 117490843648, + "segment_count_current": 527, + "segment_count_peak": 527, + "timestamp_utc": "2026-08-01T12:48:53.951621Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8639267840, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11675852800, + "inactive_split_bytes_peak_delta": 963087360, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118195486720, + "reserved_bytes_current_delta": 704643072, + "reserved_bytes_peak": 118195486720, + "reserved_bytes_peak_delta": 704643072, + "segment_count_current": 530, + "segment_count_current_delta": 3, + "segment_count_peak": 530, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T124854963026Z.pickle", + "timestamp_utc": "2026-08-01T12:48:55.071210Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T124854963026Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T124854963026Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..67be9a46101cfca67c2edb2d52e9d87e38fe3678 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T124854963026Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:989785bc5ce275fa3af8550b29365d692b94b44862291b8802b74f5eeb759b7e +size 5550228 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T124856971753Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T124856971753Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04c49ce0bedfa0cfc90323d4b3bc70fefed9961d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T124856971753Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 147721004032, + "active_bytes_current_delta": 80799885312, + "active_bytes_peak": 165293910016, + "active_bytes_peak_delta": 51981815296, + "after": { + "active_bytes_current": 147721004032, + "active_bytes_peak": 165293910016, + "allocated_bytes_current": 79303566336, + "allocated_bytes_peak": 113320822272, + "device": 6, + "inactive_split_bytes_current": 15357729792, + "inactive_split_bytes_peak": 16081032192, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 182861168640, + "reserved_bytes_peak": 182861168640, + "segment_count_current": 888, + "segment_count_peak": 888, + "timestamp_utc": "2026-08-01T12:48:56.971248Z", + "world_size": 8 + }, + "allocated_bytes_current": 79303566336, + "allocated_bytes_current_delta": 12382447616, + "allocated_bytes_peak": 113320822272, + "allocated_bytes_peak_delta": 8727552, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 113312094720, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 113312094720, + "device": 6, + "inactive_split_bytes_current": 8639267840, + "inactive_split_bytes_peak": 11675852800, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118195486720, + "reserved_bytes_peak": 118195486720, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T12:48:55.086375Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 15357729792, + "inactive_split_bytes_current_delta": 6718461952, + "inactive_split_bytes_peak": 16081032192, + "inactive_split_bytes_peak_delta": 4405179392, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 182861168640, + "reserved_bytes_current_delta": 64665681920, + "reserved_bytes_peak": 182861168640, + "reserved_bytes_peak_delta": 64665681920, + "segment_count_current": 888, + "segment_count_current_delta": 358, + "segment_count_peak": 888, + "segment_count_peak_delta": 358, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T124856971753Z.pickle", + "timestamp_utc": "2026-08-01T12:48:57.094296Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T124856971753Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T124856971753Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7302e8c613fc8923b26eea48bb692dd5721f08d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T124856971753Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a8dc3ff12c101a414b6edae579962229452c0160790857c8207598c94df00fb +size 6499951 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T124852650002Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T124852650002Z.json new file mode 100644 index 0000000000000000000000000000000000000000..19ee43e01c1b2468d863f8dd8087473d4b765b99 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T124852650002Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 31689664000, + "active_bytes_peak": 89681514496, + "active_bytes_peak_delta": 54215178240, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 89681514496, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 89681514496, + "device": 7, + "inactive_split_bytes_current": 9096446464, + "inactive_split_bytes_peak": 11006366720, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 98440314880, + "reserved_bytes_peak": 98440314880, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.649494Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 31689664000, + "allocated_bytes_peak": 89681514496, + "allocated_bytes_peak_delta": 54215178240, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:47:45.020362Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9096446464, + "inactive_split_bytes_current_delta": 8963627520, + "inactive_split_bytes_peak": 11006366720, + "inactive_split_bytes_peak_delta": 10845980672, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 98440314880, + "reserved_bytes_current_delta": 59196309504, + "reserved_bytes_peak": 98440314880, + "reserved_bytes_peak_delta": 59196309504, + "segment_count_current": 468, + "segment_count_current_delta": 397, + "segment_count_peak": 468, + "segment_count_peak_delta": 397, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T124852650002Z.pickle", + "timestamp_utc": "2026-08-01T12:48:52.759133Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T124852650002Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T124852650002Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6995e701996c1a77ddb514b6f937ab9b4987fddc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T124852650002Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab4366968106319306edbb973189986846fc230667a71d458bad3e4dac73db33 +size 5397214 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T124853830735Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T124853830735Z.json new file mode 100644 index 0000000000000000000000000000000000000000..375fe0233a2d13a616c6b52e2d568fd896fac679 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T124853830735Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113306983936, + "active_bytes_peak_delta": 23625469440, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 113306983936, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 113306983936, + "device": 7, + "inactive_split_bytes_current": 8876245504, + "inactive_split_bytes_peak": 11006366720, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117490843648, + "reserved_bytes_peak": 117490843648, + "segment_count_current": 527, + "segment_count_peak": 527, + "timestamp_utc": "2026-08-01T12:48:53.830330Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113306983936, + "allocated_bytes_peak_delta": 23625469440, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 89681514496, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 89681514496, + "device": 7, + "inactive_split_bytes_current": 9096446464, + "inactive_split_bytes_peak": 11006366720, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 98440314880, + "reserved_bytes_peak": 98440314880, + "segment_count_current": 468, + "segment_count_peak": 468, + "timestamp_utc": "2026-08-01T12:48:52.776889Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8876245504, + "inactive_split_bytes_current_delta": -220200960, + "inactive_split_bytes_peak": 11006366720, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117490843648, + "reserved_bytes_current_delta": 19050528768, + "reserved_bytes_peak": 117490843648, + "reserved_bytes_peak_delta": 19050528768, + "segment_count_current": 527, + "segment_count_current_delta": 59, + "segment_count_peak": 527, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T124853830735Z.pickle", + "timestamp_utc": "2026-08-01T12:48:53.939584Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T124853830735Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T124853830735Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..47f78d2232c5616a328e713b565da91eee818e27 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T124853830735Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53fea8a1c7a6dc5f7f087e8a08a147b6c5956984df8eab44eacc17277aec4e83 +size 5556059 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T124854963123Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T124854963123Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a420266d8edad667ae16899b4996420fd5c97756 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T124854963123Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113308990464, + "active_bytes_peak_delta": 2006528, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 113308990464, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 113308990464, + "device": 7, + "inactive_split_bytes_current": 8876245504, + "inactive_split_bytes_peak": 11006366720, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118195486720, + "reserved_bytes_peak": 118195486720, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T12:48:54.962722Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113308990464, + "allocated_bytes_peak_delta": 2006528, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 113306983936, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 113306983936, + "device": 7, + "inactive_split_bytes_current": 8876245504, + "inactive_split_bytes_peak": 11006366720, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117490843648, + "reserved_bytes_peak": 117490843648, + "segment_count_current": 527, + "segment_count_peak": 527, + "timestamp_utc": "2026-08-01T12:48:53.955458Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8876245504, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11006366720, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118195486720, + "reserved_bytes_current_delta": 704643072, + "reserved_bytes_peak": 118195486720, + "reserved_bytes_peak_delta": 704643072, + "segment_count_current": 530, + "segment_count_current_delta": 3, + "segment_count_peak": 530, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T124854963123Z.pickle", + "timestamp_utc": "2026-08-01T12:48:55.061625Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T124854963123Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T124854963123Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..20de0b7c62d2d274fa1ad09750e21df831828469 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T124854963123Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fbb90d92d156200f70b9fbb3d14f8885d1faba638dcac982b7f1b38bef7d101 +size 5558014 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T124856733826Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T124856733826Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d49888a3ec9ea8fb43092314932294b61807b59 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T124856733826Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 147488876032, + "active_bytes_current_delta": 80567756800, + "active_bytes_peak": 165295221248, + "active_bytes_peak_delta": 51986230784, + "after": { + "active_bytes_current": 147488876032, + "active_bytes_peak": 165295221248, + "allocated_bytes_current": 79071438336, + "allocated_bytes_peak": 113320822784, + "device": 7, + "inactive_split_bytes_current": 15988316672, + "inactive_split_bytes_peak": 16464853504, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 182863265792, + "reserved_bytes_peak": 182863265792, + "segment_count_current": 889, + "segment_count_peak": 889, + "timestamp_utc": "2026-08-01T12:48:56.733332Z", + "world_size": 8 + }, + "allocated_bytes_current": 79071438336, + "allocated_bytes_current_delta": 12150319104, + "allocated_bytes_peak": 113320822784, + "allocated_bytes_peak_delta": 11832320, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 113308990464, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 113308990464, + "device": 7, + "inactive_split_bytes_current": 8876245504, + "inactive_split_bytes_peak": 11006366720, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118195486720, + "reserved_bytes_peak": 118195486720, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T12:48:55.074484Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 15988316672, + "inactive_split_bytes_current_delta": 7112071168, + "inactive_split_bytes_peak": 16464853504, + "inactive_split_bytes_peak_delta": 5458486784, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 182863265792, + "reserved_bytes_current_delta": 64667779072, + "reserved_bytes_peak": 182863265792, + "reserved_bytes_peak_delta": 64667779072, + "segment_count_current": 889, + "segment_count_current_delta": 359, + "segment_count_peak": 889, + "segment_count_peak_delta": 359, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T124856733826Z.pickle", + "timestamp_utc": "2026-08-01T12:48:56.848348Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T124856733826Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T124856733826Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..851e6a8354d4eeb55e54b547733048306714dd50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T124856733826Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b62e492954cef6cdbfb3baea233d56a32f9f12439c9d7b80a49d545c4d6b85ff +size 6500689 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T125106902980Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T125106902980Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6114dad8be8d8b1435bc4d9ac098e4b8a6cf61d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T125106902980Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507874304, + "active_bytes_current_delta": 37276419072, + "active_bytes_peak": 90265517056, + "active_bytes_peak_delta": 54799180800, + "after": { + "active_bytes_current": 72507874304, + "active_bytes_peak": 90265517056, + "allocated_bytes_current": 72507874304, + "allocated_bytes_peak": 89578721280, + "device": 0, + "inactive_split_bytes_current": 8802903040, + "inactive_split_bytes_peak": 12506667008, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 99629400064, + "reserved_bytes_peak": 99629400064, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T12:51:06.902422Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507874304, + "allocated_bytes_current_delta": 37276419072, + "allocated_bytes_peak": 89578721280, + "allocated_bytes_peak_delta": 54112385024, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:49:58.743899Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8802903040, + "inactive_split_bytes_current_delta": 8667986944, + "inactive_split_bytes_peak": 12506667008, + "inactive_split_bytes_peak_delta": 12346035200, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 99629400064, + "reserved_bytes_current_delta": 60383297536, + "reserved_bytes_peak": 99629400064, + "reserved_bytes_peak_delta": 60383297536, + "segment_count_current": 422, + "segment_count_current_delta": 350, + "segment_count_peak": 422, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T125106902980Z.pickle", + "timestamp_utc": "2026-08-01T12:51:07.020805Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T125106902980Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T125106902980Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b600bb0c289b81135d83de45c3bb632bc11d30d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T125106902980Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebd5e4e870523c35189c687997b6e96b9bba2da509e1ea2700ff299597be3657 +size 6993476 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T125108195244Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T125108195244Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8845445910282e18232caa8f465e9ae2c8525064 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T125108195244Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507874304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113911515648, + "active_bytes_peak_delta": 23645998592, + "after": { + "active_bytes_current": 72507874304, + "active_bytes_peak": 113911515648, + "allocated_bytes_current": 72507874304, + "allocated_bytes_peak": 113206991360, + "device": 0, + "inactive_split_bytes_current": 11038467072, + "inactive_split_bytes_peak": 15341027840, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 120290541568, + "reserved_bytes_peak": 120290541568, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:51:08.194844Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507874304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113206991360, + "allocated_bytes_peak_delta": 23628270080, + "before": { + "active_bytes_current": 72507874304, + "active_bytes_peak": 90265517056, + "allocated_bytes_current": 72507874304, + "allocated_bytes_peak": 89578721280, + "device": 0, + "inactive_split_bytes_current": 8802903040, + "inactive_split_bytes_peak": 12506667008, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 99629400064, + "reserved_bytes_peak": 99629400064, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T12:51:07.037719Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11038467072, + "inactive_split_bytes_current_delta": 2235564032, + "inactive_split_bytes_peak": 15341027840, + "inactive_split_bytes_peak_delta": 2834360832, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 120290541568, + "reserved_bytes_current_delta": 20661141504, + "reserved_bytes_peak": 120290541568, + "reserved_bytes_peak_delta": 20661141504, + "segment_count_current": 488, + "segment_count_current_delta": 66, + "segment_count_peak": 488, + "segment_count_peak_delta": 66, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T125108195244Z.pickle", + "timestamp_utc": "2026-08-01T12:51:08.305417Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T125108195244Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T125108195244Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f25bd690e5c904acb54c8f75a9586c950031b85 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T125108195244Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc5d31ce02c93126a53167c31f9ee44c9211479de953c0f9ca66220fdf8f47c0 +size 6238663 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T125109469191Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T125109469191Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e680bda0a943a3ab0b383b4d7fdd0f2bb63c55bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T125109469191Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507874304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113911515648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72507874304, + "active_bytes_peak": 113911515648, + "allocated_bytes_current": 72507874304, + "allocated_bytes_peak": 113206991360, + "device": 0, + "inactive_split_bytes_current": 10921026560, + "inactive_split_bytes_peak": 15341027840, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 120290541568, + "reserved_bytes_peak": 120290541568, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:51:09.468796Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507874304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113206991360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507874304, + "active_bytes_peak": 113911515648, + "allocated_bytes_current": 72507874304, + "allocated_bytes_peak": 113206991360, + "device": 0, + "inactive_split_bytes_current": 11038467072, + "inactive_split_bytes_peak": 15341027840, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 120290541568, + "reserved_bytes_peak": 120290541568, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:51:08.319501Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10921026560, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 15341027840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 120290541568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120290541568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T125109469191Z.pickle", + "timestamp_utc": "2026-08-01T12:51:09.575316Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T125109469191Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T125109469191Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ab303669be96e9219391f1d5ea36cd45672e6bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T125109469191Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13c7fabf3cfc95f766babbcb8de2f2fdc779b8950d1f543175cfde15bd6f55f0 +size 6035171 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T125111177251Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T125111177251Z.json new file mode 100644 index 0000000000000000000000000000000000000000..821f6658fc1e081b3bd58e6c162e3c7ee657fb1a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T125111177251Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 161408664064, + "active_bytes_current_delta": 88900789760, + "active_bytes_peak": 161810070528, + "active_bytes_peak_delta": 47898554880, + "after": { + "active_bytes_current": 161408664064, + "active_bytes_peak": 161810070528, + "allocated_bytes_current": 105206276608, + "allocated_bytes_peak": 113217963520, + "device": 0, + "inactive_split_bytes_current": 14582234624, + "inactive_split_bytes_peak": 22204163072, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 182921986048, + "reserved_bytes_peak": 182921986048, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:51:11.176563Z", + "world_size": 8 + }, + "allocated_bytes_current": 105206276608, + "allocated_bytes_current_delta": 32698402304, + "allocated_bytes_peak": 113217963520, + "allocated_bytes_peak_delta": 10972160, + "before": { + "active_bytes_current": 72507874304, + "active_bytes_peak": 113911515648, + "allocated_bytes_current": 72507874304, + "allocated_bytes_peak": 113206991360, + "device": 0, + "inactive_split_bytes_current": 10921026560, + "inactive_split_bytes_peak": 15341027840, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 120290541568, + "reserved_bytes_peak": 120290541568, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:51:09.589074Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 14582234624, + "inactive_split_bytes_current_delta": 3661208064, + "inactive_split_bytes_peak": 22204163072, + "inactive_split_bytes_peak_delta": 6863135232, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 182921986048, + "reserved_bytes_current_delta": 62631444480, + "reserved_bytes_peak": 182921986048, + "reserved_bytes_peak_delta": 62631444480, + "segment_count_current": 747, + "segment_count_current_delta": 259, + "segment_count_peak": 747, + "segment_count_peak_delta": 259, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T125111177251Z.pickle", + "timestamp_utc": "2026-08-01T12:51:11.279433Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T125111177251Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T125111177251Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..37ddf436587acf4da2dc355c286c080c14233bf4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T125111177251Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7479be81c523322cf7d5d65487bb99e99b928793b9c3bc5f056e4c60143a888f +size 5438986 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T125106903873Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T125106903873Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ebedc3bd68be901abf0d28d52e0d09c18068948 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T125106903873Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508365824, + "active_bytes_current_delta": 37276910592, + "active_bytes_peak": 90279517696, + "active_bytes_peak_delta": 54813181440, + "after": { + "active_bytes_current": 72508365824, + "active_bytes_peak": 90279517696, + "allocated_bytes_current": 72508365824, + "allocated_bytes_peak": 89577222144, + "device": 1, + "inactive_split_bytes_current": 8770954240, + "inactive_split_bytes_peak": 11726789632, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 99803463680, + "reserved_bytes_peak": 99803463680, + "segment_count_current": 419, + "segment_count_peak": 419, + "timestamp_utc": "2026-08-01T12:51:06.903223Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508365824, + "allocated_bytes_current_delta": 37276910592, + "allocated_bytes_peak": 89577222144, + "allocated_bytes_peak_delta": 54110885888, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:49:58.743834Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8770954240, + "inactive_split_bytes_current_delta": 8638135296, + "inactive_split_bytes_peak": 11726789632, + "inactive_split_bytes_peak_delta": 11566403584, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 99803463680, + "reserved_bytes_current_delta": 60559458304, + "reserved_bytes_peak": 99803463680, + "reserved_bytes_peak_delta": 60559458304, + "segment_count_current": 419, + "segment_count_current_delta": 348, + "segment_count_peak": 419, + "segment_count_peak_delta": 348, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T125106903873Z.pickle", + "timestamp_utc": "2026-08-01T12:51:07.025194Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T125106903873Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T125106903873Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f51cb7da796b7e9b0ee1b35f20d4c7528ae03b00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T125106903873Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dbac3427b9c879d40f01661ec8be0e30382f2ddd7045606666644953402b42a +size 5956090 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T125108195277Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T125108195277Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0539ef859c7648c33b31df7a56afbacdbc8485e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T125108195277Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508365824, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113917569536, + "active_bytes_peak_delta": 23638051840, + "after": { + "active_bytes_current": 72508365824, + "active_bytes_peak": 113917569536, + "allocated_bytes_current": 72508365824, + "allocated_bytes_peak": 113213045248, + "device": 1, + "inactive_split_bytes_current": 14114497536, + "inactive_split_bytes_peak": 14173217792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120435245056, + "reserved_bytes_peak": 120435245056, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T12:51:08.194874Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508365824, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113213045248, + "allocated_bytes_peak_delta": 23635823104, + "before": { + "active_bytes_current": 72508365824, + "active_bytes_peak": 90279517696, + "allocated_bytes_current": 72508365824, + "allocated_bytes_peak": 89577222144, + "device": 1, + "inactive_split_bytes_current": 8770954240, + "inactive_split_bytes_peak": 11726789632, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 99803463680, + "reserved_bytes_peak": 99803463680, + "segment_count_current": 419, + "segment_count_peak": 419, + "timestamp_utc": "2026-08-01T12:51:07.045977Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 14114497536, + "inactive_split_bytes_current_delta": 5343543296, + "inactive_split_bytes_peak": 14173217792, + "inactive_split_bytes_peak_delta": 2446428160, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120435245056, + "reserved_bytes_current_delta": 20631781376, + "reserved_bytes_peak": 120435245056, + "reserved_bytes_peak_delta": 20631781376, + "segment_count_current": 483, + "segment_count_current_delta": 64, + "segment_count_peak": 483, + "segment_count_peak_delta": 64, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T125108195277Z.pickle", + "timestamp_utc": "2026-08-01T12:51:08.304015Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T125108195277Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T125108195277Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1330589af3c1db8dcaf7cd743e4b8a14c3e78001 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T125108195277Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5bd90b88f3ddf377857f5660ed445365d08b7dad6ed3807d38314651a80d09f +size 5920771 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T125109469197Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T125109469197Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e0b33c941d7a7387bd32d7fbdb6d9fa511b32d3a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T125109469197Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508365824, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113917569536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72508365824, + "active_bytes_peak": 113917569536, + "allocated_bytes_current": 72508365824, + "allocated_bytes_peak": 113213045248, + "device": 1, + "inactive_split_bytes_current": 12822651904, + "inactive_split_bytes_peak": 15334886912, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 121261522944, + "reserved_bytes_peak": 121261522944, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T12:51:09.468781Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508365824, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113213045248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72508365824, + "active_bytes_peak": 113917569536, + "allocated_bytes_current": 72508365824, + "allocated_bytes_peak": 113213045248, + "device": 1, + "inactive_split_bytes_current": 14114497536, + "inactive_split_bytes_peak": 14173217792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 120435245056, + "reserved_bytes_peak": 120435245056, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T12:51:08.319985Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12822651904, + "inactive_split_bytes_current_delta": -1291845632, + "inactive_split_bytes_peak": 15334886912, + "inactive_split_bytes_peak_delta": 1161669120, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 121261522944, + "reserved_bytes_current_delta": 826277888, + "reserved_bytes_peak": 121261522944, + "reserved_bytes_peak_delta": 826277888, + "segment_count_current": 485, + "segment_count_current_delta": 2, + "segment_count_peak": 485, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T125109469197Z.pickle", + "timestamp_utc": "2026-08-01T12:51:09.578454Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T125109469197Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T125109469197Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..370cc37e8e204215cf485451ea288456d3fba413 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T125109469197Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb491c23c00d21469a7fc412d5c8533098959b477fc98ff09b694f7a66674c88 +size 6075280 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T125111184255Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T125111184255Z.json new file mode 100644 index 0000000000000000000000000000000000000000..46f02ec93dfece79ad0e707b6e653b6e11ed6647 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T125111184255Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 158254396416, + "active_bytes_current_delta": 85746030592, + "active_bytes_peak": 158254396416, + "active_bytes_peak_delta": 44336826880, + "after": { + "active_bytes_current": 158254396416, + "active_bytes_peak": 158254396416, + "allocated_bytes_current": 102756652032, + "allocated_bytes_peak": 113214785024, + "device": 1, + "inactive_split_bytes_current": 16950070272, + "inactive_split_bytes_peak": 23600366592, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 182504652800, + "reserved_bytes_peak": 182504652800, + "segment_count_current": 739, + "segment_count_peak": 739, + "timestamp_utc": "2026-08-01T12:51:11.183639Z", + "world_size": 8 + }, + "allocated_bytes_current": 102756652032, + "allocated_bytes_current_delta": 30248286208, + "allocated_bytes_peak": 113214785024, + "allocated_bytes_peak_delta": 1739776, + "before": { + "active_bytes_current": 72508365824, + "active_bytes_peak": 113917569536, + "allocated_bytes_current": 72508365824, + "allocated_bytes_peak": 113213045248, + "device": 1, + "inactive_split_bytes_current": 12822651904, + "inactive_split_bytes_peak": 15334886912, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 121261522944, + "reserved_bytes_peak": 121261522944, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T12:51:09.594311Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 16950070272, + "inactive_split_bytes_current_delta": 4127418368, + "inactive_split_bytes_peak": 23600366592, + "inactive_split_bytes_peak_delta": 8265479680, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 182504652800, + "reserved_bytes_current_delta": 61243129856, + "reserved_bytes_peak": 182504652800, + "reserved_bytes_peak_delta": 61243129856, + "segment_count_current": 739, + "segment_count_current_delta": 254, + "segment_count_peak": 739, + "segment_count_peak_delta": 254, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T125111184255Z.pickle", + "timestamp_utc": "2026-08-01T12:51:11.288901Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T125111184255Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T125111184255Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8daf17c17ebd1d29447f757bcbed2acd789e7a40 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T125111184255Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26aded89dfaccce0508f6a1a8ebfa1622ff35a9300190cd92a0b8b4cd8dc587e +size 5278590 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T125106902980Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T125106902980Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a33fcfafd0ba342da48a0414fcfce5511f38d9c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T125106902980Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 90283318784, + "active_bytes_peak_delta": 54816982528, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 90283318784, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 89579974656, + "device": 2, + "inactive_split_bytes_current": 8889140224, + "inactive_split_bytes_peak": 11725140992, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 99568582656, + "reserved_bytes_peak": 99568582656, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T12:51:06.902402Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 89579974656, + "allocated_bytes_peak_delta": 54113638400, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:49:58.744444Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8889140224, + "inactive_split_bytes_current_delta": 8756321280, + "inactive_split_bytes_peak": 11725140992, + "inactive_split_bytes_peak_delta": 11564754944, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 99568582656, + "reserved_bytes_current_delta": 60324577280, + "reserved_bytes_peak": 99568582656, + "reserved_bytes_peak_delta": 60324577280, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T125106902980Z.pickle", + "timestamp_utc": "2026-08-01T12:51:07.021387Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T125106902980Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T125106902980Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e6ef4920dfbeb9f16b53703b775aaafe97becb1e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T125106902980Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbc7f1f8df888670274eb1363f57de9324b841e1a4fec712c192599f3869dce0 +size 5955057 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T125108195370Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T125108195370Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e6c6e555ce84c53db2a644b779e0754e03e6362 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T125108195370Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113558543360, + "active_bytes_peak_delta": 23275224576, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113558543360, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113206221824, + "device": 2, + "inactive_split_bytes_current": 12057936896, + "inactive_split_bytes_peak": 14284962304, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120756109312, + "reserved_bytes_peak": 120756109312, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:51:08.194900Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113206221824, + "allocated_bytes_peak_delta": 23626247168, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 90283318784, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 89579974656, + "device": 2, + "inactive_split_bytes_current": 8889140224, + "inactive_split_bytes_peak": 11725140992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 99568582656, + "reserved_bytes_peak": 99568582656, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T12:51:07.040581Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12057936896, + "inactive_split_bytes_current_delta": 3168796672, + "inactive_split_bytes_peak": 14284962304, + "inactive_split_bytes_peak_delta": 2559821312, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120756109312, + "reserved_bytes_current_delta": 21187526656, + "reserved_bytes_peak": 120756109312, + "reserved_bytes_peak_delta": 21187526656, + "segment_count_current": 487, + "segment_count_current_delta": 67, + "segment_count_peak": 487, + "segment_count_peak_delta": 67, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T125108195370Z.pickle", + "timestamp_utc": "2026-08-01T12:51:08.310856Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T125108195370Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T125108195370Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4afaf118cd3ba5585b8f7ab1b01dc644e2079388 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T125108195370Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fff7d6be6a1745b8b24dbf6c26531cbf26c008c19f18d93dd32ac736c221491b +size 6058545 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T125109469270Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T125109469270Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7be6c3ffac77a83464f029023b3d64fd31b22634 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T125109469270Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113907243520, + "active_bytes_peak_delta": 348700160, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113907243520, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113206221824, + "device": 2, + "inactive_split_bytes_current": 12471075840, + "inactive_split_bytes_peak": 14298969600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 121582387200, + "reserved_bytes_peak": 121582387200, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T12:51:09.468817Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113206221824, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113558543360, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113206221824, + "device": 2, + "inactive_split_bytes_current": 12057936896, + "inactive_split_bytes_peak": 14284962304, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120756109312, + "reserved_bytes_peak": 120756109312, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:51:08.327594Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12471075840, + "inactive_split_bytes_current_delta": 413138944, + "inactive_split_bytes_peak": 14298969600, + "inactive_split_bytes_peak_delta": 14007296, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 121582387200, + "reserved_bytes_current_delta": 826277888, + "reserved_bytes_peak": 121582387200, + "reserved_bytes_peak_delta": 826277888, + "segment_count_current": 489, + "segment_count_current_delta": 2, + "segment_count_peak": 489, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T125109469270Z.pickle", + "timestamp_utc": "2026-08-01T12:51:09.580607Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T125109469270Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T125109469270Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a54071638f616748c15d1b20c6db4e940a56eedf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T125109469270Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb20199968a5e00517419447926e2bc6f11e58ea7a04184569edd8c5954b811b +size 6078992 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T125111306896Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T125111306896Z.json new file mode 100644 index 0000000000000000000000000000000000000000..903c04cceded85da787e9695f9c307527d5cd62c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T125111306896Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 159087924224, + "active_bytes_current_delta": 86580303872, + "active_bytes_peak": 159239312384, + "active_bytes_peak_delta": 45332068864, + "after": { + "active_bytes_current": 159087924224, + "active_bytes_peak": 159239312384, + "allocated_bytes_current": 103590179840, + "allocated_bytes_peak": 113221641728, + "device": 2, + "inactive_split_bytes_current": 17024609280, + "inactive_split_bytes_peak": 24217936896, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 182827614208, + "reserved_bytes_peak": 182827614208, + "segment_count_current": 739, + "segment_count_peak": 739, + "timestamp_utc": "2026-08-01T12:51:11.306237Z", + "world_size": 8 + }, + "allocated_bytes_current": 103590179840, + "allocated_bytes_current_delta": 31082559488, + "allocated_bytes_peak": 113221641728, + "allocated_bytes_peak_delta": 15419904, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113907243520, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113206221824, + "device": 2, + "inactive_split_bytes_current": 12471075840, + "inactive_split_bytes_peak": 14298969600, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 121582387200, + "reserved_bytes_peak": 121582387200, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T12:51:09.596680Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 17024609280, + "inactive_split_bytes_current_delta": 4553533440, + "inactive_split_bytes_peak": 24217936896, + "inactive_split_bytes_peak_delta": 9918967296, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 182827614208, + "reserved_bytes_current_delta": 61245227008, + "reserved_bytes_peak": 182827614208, + "reserved_bytes_peak_delta": 61245227008, + "segment_count_current": 739, + "segment_count_current_delta": 250, + "segment_count_peak": 739, + "segment_count_peak_delta": 250, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T125111306896Z.pickle", + "timestamp_utc": "2026-08-01T12:51:11.412514Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T125111306896Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T125111306896Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..559a9b541b1a0143639daafbc60952f6a476de4c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T125111306896Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f3114e35b9547d701cdec624c5d3ed3e20cb41719dcf4240da9e015cab9a0cc +size 5276574 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T125106903358Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T125106903358Z.json new file mode 100644 index 0000000000000000000000000000000000000000..81ba0e73167d74815b8a5fa41f9c9b1cc1d0b5fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T125106903358Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 89926004224, + "active_bytes_peak_delta": 54459667968, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 89926004224, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 89573682688, + "device": 3, + "inactive_split_bytes_current": 8538915840, + "inactive_split_bytes_peak": 12020368384, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 99040100352, + "reserved_bytes_peak": 99040100352, + "segment_count_current": 418, + "segment_count_peak": 418, + "timestamp_utc": "2026-08-01T12:51:06.902708Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 89573682688, + "allocated_bytes_peak_delta": 54107346432, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:49:58.743660Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8538915840, + "inactive_split_bytes_current_delta": 8406096896, + "inactive_split_bytes_peak": 12020368384, + "inactive_split_bytes_peak_delta": 11859982336, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 99040100352, + "reserved_bytes_current_delta": 59796094976, + "reserved_bytes_peak": 99040100352, + "reserved_bytes_peak_delta": 59796094976, + "segment_count_current": 418, + "segment_count_current_delta": 347, + "segment_count_peak": 418, + "segment_count_peak_delta": 347, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T125106903358Z.pickle", + "timestamp_utc": "2026-08-01T12:51:07.024463Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T125106903358Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T125106903358Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db6e8caad168a6e06edb1f28ab7dcc4f961ce641 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T125106903358Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b59005a97b2323dafdb20172725acf5a37de3cb0908c94d6d3e92d2b4e7dafa8 +size 5939067 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T125108195487Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T125108195487Z.json new file mode 100644 index 0000000000000000000000000000000000000000..90f683492b71eac4c46b519e60a7d9b2d0385743 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T125108195487Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113557109760, + "active_bytes_peak_delta": 23631105536, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113557109760, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113204788224, + "device": 3, + "inactive_split_bytes_current": 12705956864, + "inactive_split_bytes_peak": 14008688128, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120342970368, + "reserved_bytes_peak": 120342970368, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T12:51:08.194995Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113204788224, + "allocated_bytes_peak_delta": 23631105536, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 89926004224, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 89573682688, + "device": 3, + "inactive_split_bytes_current": 8538915840, + "inactive_split_bytes_peak": 12020368384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 99040100352, + "reserved_bytes_peak": 99040100352, + "segment_count_current": 418, + "segment_count_peak": 418, + "timestamp_utc": "2026-08-01T12:51:07.041849Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12705956864, + "inactive_split_bytes_current_delta": 4167041024, + "inactive_split_bytes_peak": 14008688128, + "inactive_split_bytes_peak_delta": 1988319744, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120342970368, + "reserved_bytes_current_delta": 21302870016, + "reserved_bytes_peak": 120342970368, + "reserved_bytes_peak_delta": 21302870016, + "segment_count_current": 484, + "segment_count_current_delta": 66, + "segment_count_peak": 484, + "segment_count_peak_delta": 66, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T125108195487Z.pickle", + "timestamp_utc": "2026-08-01T12:51:08.307737Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T125108195487Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T125108195487Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ab2824197031077d9e56b0350bdfccd2846eaa5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T125108195487Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b616d7bd2e5e069d838cea44987c64452adc913e20d4204485c61aeb057ba8fa +size 5921101 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T125109469481Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T125109469481Z.json new file mode 100644 index 0000000000000000000000000000000000000000..beafc54001dd123855b860b3b11e15701d79f31e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T125109469481Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113909299200, + "active_bytes_peak_delta": 352189440, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113909299200, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113210424320, + "device": 3, + "inactive_split_bytes_current": 11646895104, + "inactive_split_bytes_peak": 14277819904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120342970368, + "reserved_bytes_peak": 120342970368, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T12:51:09.468948Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113210424320, + "allocated_bytes_peak_delta": 5636096, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113557109760, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113204788224, + "device": 3, + "inactive_split_bytes_current": 12705956864, + "inactive_split_bytes_peak": 14008688128, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 120342970368, + "reserved_bytes_peak": 120342970368, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T12:51:08.322259Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 11646895104, + "inactive_split_bytes_current_delta": -1059061760, + "inactive_split_bytes_peak": 14277819904, + "inactive_split_bytes_peak_delta": 269131776, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120342970368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120342970368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 484, + "segment_count_current_delta": 0, + "segment_count_peak": 484, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T125109469481Z.pickle", + "timestamp_utc": "2026-08-01T12:51:09.580275Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T125109469481Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T125109469481Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94a7361de3d021932e80099ebbf41c6c09092077 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T125109469481Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a81de834834621a6efa98e15d85f0ed0f2e123b51aa1f35c630ab5a86e200ba +size 5918231 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T125111388121Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T125111388121Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4c6b04ead94681d9ff19081faf5d43ff507be28 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T125111388121Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 160881957888, + "active_bytes_current_delta": 88374337536, + "active_bytes_peak": 160934910976, + "active_bytes_peak_delta": 47025611776, + "after": { + "active_bytes_current": 160881957888, + "active_bytes_peak": 160934910976, + "allocated_bytes_current": 105031891968, + "allocated_bytes_peak": 113218627072, + "device": 3, + "inactive_split_bytes_current": 16453215232, + "inactive_split_bytes_peak": 23952917504, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 182741630976, + "reserved_bytes_peak": 182741630976, + "segment_count_current": 742, + "segment_count_peak": 742, + "timestamp_utc": "2026-08-01T12:51:11.387449Z", + "world_size": 8 + }, + "allocated_bytes_current": 105031891968, + "allocated_bytes_current_delta": 32524271616, + "allocated_bytes_peak": 113218627072, + "allocated_bytes_peak_delta": 8202752, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113909299200, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113210424320, + "device": 3, + "inactive_split_bytes_current": 11646895104, + "inactive_split_bytes_peak": 14277819904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 120342970368, + "reserved_bytes_peak": 120342970368, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T12:51:09.594477Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 16453215232, + "inactive_split_bytes_current_delta": 4806320128, + "inactive_split_bytes_peak": 23952917504, + "inactive_split_bytes_peak_delta": 9675097600, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 182741630976, + "reserved_bytes_current_delta": 62398660608, + "reserved_bytes_peak": 182741630976, + "reserved_bytes_peak_delta": 62398660608, + "segment_count_current": 742, + "segment_count_current_delta": 258, + "segment_count_peak": 742, + "segment_count_peak_delta": 258, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T125111388121Z.pickle", + "timestamp_utc": "2026-08-01T12:51:11.494680Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T125111388121Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T125111388121Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b678f4b64e3e6669e0440add1da2f27de03c6656 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T125111388121Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29083f0016e7cbf2b7820a87504746bad34593a1e3a107e44eb0783ae6ec66cb +size 5341328 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T125106903300Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T125106903300Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf8da19cd3b5fb6f3a3dc6024ea1d89fe040205d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T125106903300Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509203456, + "active_bytes_current_delta": 37277305856, + "active_bytes_peak": 89929805312, + "active_bytes_peak_delta": 54463026688, + "after": { + "active_bytes_current": 72509203456, + "active_bytes_peak": 89929805312, + "allocated_bytes_current": 72509203456, + "allocated_bytes_peak": 89577483776, + "device": 4, + "inactive_split_bytes_current": 9466371072, + "inactive_split_bytes_peak": 12621266944, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 99686023168, + "reserved_bytes_peak": 99686023168, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:51:06.902655Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509203456, + "allocated_bytes_current_delta": 37277305856, + "allocated_bytes_peak": 89577483776, + "allocated_bytes_peak_delta": 54110705152, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:49:58.743680Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9466371072, + "inactive_split_bytes_current_delta": 9333994496, + "inactive_split_bytes_peak": 12621266944, + "inactive_split_bytes_peak_delta": 12467857408, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 99686023168, + "reserved_bytes_current_delta": 60442017792, + "reserved_bytes_peak": 99686023168, + "reserved_bytes_peak_delta": 60442017792, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T125106903300Z.pickle", + "timestamp_utc": "2026-08-01T12:51:07.023620Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T125106903300Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T125106903300Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..923d35535f08a7da2ac044f11f03dde8c0638299 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T125106903300Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c94ca173e596f37a7448515a1d13628ba4127febfbcddf13e207b40546e2699b +size 7136892 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T125108195476Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T125108195476Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d9c7c74e3349b14ac29c95b202363cddd50a6f02 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T125108195476Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509203456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113558879232, + "active_bytes_peak_delta": 23629073920, + "after": { + "active_bytes_current": 72509203456, + "active_bytes_peak": 113558879232, + "allocated_bytes_current": 72509203456, + "allocated_bytes_peak": 113207609856, + "device": 4, + "inactive_split_bytes_current": 12287040512, + "inactive_split_bytes_peak": 13532992512, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120638668800, + "reserved_bytes_peak": 120638668800, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T12:51:08.194970Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509203456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113207609856, + "allocated_bytes_peak_delta": 23630126080, + "before": { + "active_bytes_current": 72509203456, + "active_bytes_peak": 89929805312, + "allocated_bytes_current": 72509203456, + "allocated_bytes_peak": 89577483776, + "device": 4, + "inactive_split_bytes_current": 9466371072, + "inactive_split_bytes_peak": 12621266944, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 99686023168, + "reserved_bytes_peak": 99686023168, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T12:51:07.040308Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 12287040512, + "inactive_split_bytes_current_delta": 2820669440, + "inactive_split_bytes_peak": 13532992512, + "inactive_split_bytes_peak_delta": 911725568, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120638668800, + "reserved_bytes_current_delta": 20952645632, + "reserved_bytes_peak": 120638668800, + "reserved_bytes_peak_delta": 20952645632, + "segment_count_current": 489, + "segment_count_current_delta": 68, + "segment_count_peak": 489, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T125108195476Z.pickle", + "timestamp_utc": "2026-08-01T12:51:08.307994Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T125108195476Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T125108195476Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ed7bb61daba284bed71d48af4d530c43752966c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T125108195476Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f230af856b4fe16cf4ec02cafed717d1ddd48b8896a2c3e74547c6d76c4d74d4 +size 5947257 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T125109469411Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T125109469411Z.json new file mode 100644 index 0000000000000000000000000000000000000000..082ed52f9f15973f14bae9b86e33e0537b190905 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T125109469411Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509203456, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113906755584, + "active_bytes_peak_delta": 347876352, + "after": { + "active_bytes_current": 72509203456, + "active_bytes_peak": 113906755584, + "allocated_bytes_current": 72509203456, + "allocated_bytes_peak": 113207609856, + "device": 4, + "inactive_split_bytes_current": 12580641792, + "inactive_split_bytes_peak": 14482750976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121462849536, + "reserved_bytes_peak": 121462849536, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T12:51:09.468903Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509203456, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113207609856, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72509203456, + "active_bytes_peak": 113558879232, + "allocated_bytes_current": 72509203456, + "allocated_bytes_peak": 113207609856, + "device": 4, + "inactive_split_bytes_current": 12287040512, + "inactive_split_bytes_peak": 13532992512, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120638668800, + "reserved_bytes_peak": 120638668800, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T12:51:08.321617Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 12580641792, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 14482750976, + "inactive_split_bytes_peak_delta": 949758464, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121462849536, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 121462849536, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 490, + "segment_count_current_delta": 1, + "segment_count_peak": 490, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T125109469411Z.pickle", + "timestamp_utc": "2026-08-01T12:51:09.584419Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T125109469411Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T125109469411Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..98e136d81578c120772a8a1246539e1605df415e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T125109469411Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75d7cd3c17aaae93b31100ae6e2b1e9bad37f955c99162529e464b5f47315a41 +size 5943520 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T125111254613Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T125111254613Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bcab5bce420033c89554522ba28def3bf696100a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T125111254613Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 159563904000, + "active_bytes_current_delta": 87054700544, + "active_bytes_peak": 159727303680, + "active_bytes_peak_delta": 45820548096, + "after": { + "active_bytes_current": 159563904000, + "active_bytes_peak": 159727303680, + "allocated_bytes_current": 103713838080, + "allocated_bytes_peak": 113222962688, + "device": 4, + "inactive_split_bytes_current": 17804823552, + "inactive_split_bytes_peak": 25707699200, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 182760505344, + "reserved_bytes_peak": 182760505344, + "segment_count_current": 737, + "segment_count_peak": 737, + "timestamp_utc": "2026-08-01T12:51:11.253959Z", + "world_size": 8 + }, + "allocated_bytes_current": 103713838080, + "allocated_bytes_current_delta": 31204634624, + "allocated_bytes_peak": 113222962688, + "allocated_bytes_peak_delta": 15352832, + "before": { + "active_bytes_current": 72509203456, + "active_bytes_peak": 113906755584, + "allocated_bytes_current": 72509203456, + "allocated_bytes_peak": 113207609856, + "device": 4, + "inactive_split_bytes_current": 12580641792, + "inactive_split_bytes_peak": 14482750976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121462849536, + "reserved_bytes_peak": 121462849536, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T12:51:09.603672Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 17804823552, + "inactive_split_bytes_current_delta": 5224181760, + "inactive_split_bytes_peak": 25707699200, + "inactive_split_bytes_peak_delta": 11224948224, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 182760505344, + "reserved_bytes_current_delta": 61297655808, + "reserved_bytes_peak": 182760505344, + "reserved_bytes_peak_delta": 61297655808, + "segment_count_current": 737, + "segment_count_current_delta": 247, + "segment_count_peak": 737, + "segment_count_peak_delta": 247, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T125111254613Z.pickle", + "timestamp_utc": "2026-08-01T12:51:11.357557Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T125111254613Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T125111254613Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..02d28fc0f635154529efd0bbf849c640a4c2b554 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T125111254613Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27ee81fe86ecae592bb66ea64c220e7ff83cc34e891ec46d1fe872be2c423cff +size 5387305 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T125106902958Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T125106902958Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9afd9e1e080f358cfc6ee79608e7bbbb9e373edb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T125106902958Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 90279256576, + "active_bytes_peak_delta": 54812920320, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 90279256576, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 89577484288, + "device": 5, + "inactive_split_bytes_current": 8536818688, + "inactive_split_bytes_peak": 11220663296, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 99274981376, + "reserved_bytes_peak": 99274981376, + "segment_count_current": 419, + "segment_count_peak": 419, + "timestamp_utc": "2026-08-01T12:51:06.902368Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 89577484288, + "allocated_bytes_peak_delta": 54111148032, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:49:58.743868Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8536818688, + "inactive_split_bytes_current_delta": 8403999744, + "inactive_split_bytes_peak": 11220663296, + "inactive_split_bytes_peak_delta": 11060277248, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 99274981376, + "reserved_bytes_current_delta": 60030976000, + "reserved_bytes_peak": 99274981376, + "reserved_bytes_peak_delta": 60030976000, + "segment_count_current": 419, + "segment_count_current_delta": 348, + "segment_count_peak": 419, + "segment_count_peak_delta": 348, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T125106902958Z.pickle", + "timestamp_utc": "2026-08-01T12:51:07.017342Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T125106902958Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T125106902958Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7474820f22c107759173e58cec6db78367bc66ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T125106902958Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:585827e1dfde959b674f599bae6ff7526158618834e97a54fedcf3bce8b6edae +size 5938450 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T125108195240Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T125108195240Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c8ea1afdb87ec982e084a0c0b17ecb06b92129be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T125108195240Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113560418816, + "active_bytes_peak_delta": 23281162240, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113560418816, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113208216064, + "device": 5, + "inactive_split_bytes_current": 11829347328, + "inactive_split_bytes_peak": 14253855232, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120758206464, + "reserved_bytes_peak": 120758206464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:51:08.194830Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113208216064, + "allocated_bytes_peak_delta": 23630731776, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 90279256576, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 89577484288, + "device": 5, + "inactive_split_bytes_current": 8536818688, + "inactive_split_bytes_peak": 11220663296, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 99274981376, + "reserved_bytes_peak": 99274981376, + "segment_count_current": 419, + "segment_count_peak": 419, + "timestamp_utc": "2026-08-01T12:51:07.034143Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11829347328, + "inactive_split_bytes_current_delta": 3292528640, + "inactive_split_bytes_peak": 14253855232, + "inactive_split_bytes_peak_delta": 3033191936, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120758206464, + "reserved_bytes_current_delta": 21483225088, + "reserved_bytes_peak": 120758206464, + "reserved_bytes_peak_delta": 21483225088, + "segment_count_current": 488, + "segment_count_current_delta": 69, + "segment_count_peak": 488, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T125108195240Z.pickle", + "timestamp_utc": "2026-08-01T12:51:08.303937Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T125108195240Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T125108195240Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..55ed018368b8d806b8c96e61c40b85321e65b7ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T125108195240Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81b00a0a792d8b21a88ecd2c6b9a7d6ce3866bd40e42b193986d82f7c00cc14a +size 6065062 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T125109469175Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T125109469175Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ae1b277959bbe118f705aa9061d25dca013bf45 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T125109469175Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113909140480, + "active_bytes_peak_delta": 348721664, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113909140480, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113208216064, + "device": 5, + "inactive_split_bytes_current": 11825153024, + "inactive_split_bytes_peak": 14654583296, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120758206464, + "reserved_bytes_peak": 120758206464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:51:09.468767Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113208216064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113560418816, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113208216064, + "device": 5, + "inactive_split_bytes_current": 11829347328, + "inactive_split_bytes_peak": 14253855232, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120758206464, + "reserved_bytes_peak": 120758206464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:51:08.318054Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11825153024, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 14654583296, + "inactive_split_bytes_peak_delta": 400728064, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120758206464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120758206464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T125109469175Z.pickle", + "timestamp_utc": "2026-08-01T12:51:09.574961Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T125109469175Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T125109469175Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bbd4d7cc691bb7b40d77089b08c9e008c2e72d77 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T125109469175Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2688d0e5db650db2a00835bf28037e32e9b3b848aad6cebed1a31d8fdc81ef4a +size 6076018 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T125111209692Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T125111209692Z.json new file mode 100644 index 0000000000000000000000000000000000000000..849959c83a290079958e7fec0d48fe54d358a14c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T125111209692Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 160329358336, + "active_bytes_current_delta": 87821737984, + "active_bytes_peak": 160329358336, + "active_bytes_peak_delta": 46420217856, + "after": { + "active_bytes_current": 160329358336, + "active_bytes_peak": 160329358336, + "allocated_bytes_current": 104479292416, + "allocated_bytes_peak": 113218627072, + "device": 5, + "inactive_split_bytes_current": 17240695808, + "inactive_split_bytes_peak": 24368153600, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 182827614208, + "reserved_bytes_peak": 182827614208, + "segment_count_current": 740, + "segment_count_peak": 740, + "timestamp_utc": "2026-08-01T12:51:11.209088Z", + "world_size": 8 + }, + "allocated_bytes_current": 104479292416, + "allocated_bytes_current_delta": 31971672064, + "allocated_bytes_peak": 113218627072, + "allocated_bytes_peak_delta": 10411008, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113909140480, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113208216064, + "device": 5, + "inactive_split_bytes_current": 11825153024, + "inactive_split_bytes_peak": 14654583296, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120758206464, + "reserved_bytes_peak": 120758206464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T12:51:09.588507Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 17240695808, + "inactive_split_bytes_current_delta": 5415542784, + "inactive_split_bytes_peak": 24368153600, + "inactive_split_bytes_peak_delta": 9713570304, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 182827614208, + "reserved_bytes_current_delta": 62069407744, + "reserved_bytes_peak": 182827614208, + "reserved_bytes_peak_delta": 62069407744, + "segment_count_current": 740, + "segment_count_current_delta": 252, + "segment_count_peak": 740, + "segment_count_peak_delta": 252, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T125111209692Z.pickle", + "timestamp_utc": "2026-08-01T12:51:11.311784Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T125111209692Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T125111209692Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a0c6aaf99ac6545089d9e67689c57b7c80b33a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T125111209692Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2acfa44387b46955a2e208d6d567919709b01192d9bf35ae5cb7525c2dde207f +size 5341112 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T125106902945Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T125106902945Z.json new file mode 100644 index 0000000000000000000000000000000000000000..665eb61f14548a46ba96eb807bc5fabd5e99e11b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T125106902945Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 90269150208, + "active_bytes_peak_delta": 54802813952, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 90269150208, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 89574993408, + "device": 6, + "inactive_split_bytes_current": 8595538944, + "inactive_split_bytes_peak": 11273692160, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 99067363328, + "reserved_bytes_peak": 99067363328, + "segment_count_current": 417, + "segment_count_peak": 417, + "timestamp_utc": "2026-08-01T12:51:06.902366Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 89574993408, + "allocated_bytes_peak_delta": 54108657152, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:49:58.744408Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8595538944, + "inactive_split_bytes_current_delta": 8462720000, + "inactive_split_bytes_peak": 11273692160, + "inactive_split_bytes_peak_delta": 11113306112, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 99067363328, + "reserved_bytes_current_delta": 59823357952, + "reserved_bytes_peak": 99067363328, + "reserved_bytes_peak_delta": 59823357952, + "segment_count_current": 417, + "segment_count_current_delta": 346, + "segment_count_peak": 417, + "segment_count_peak_delta": 346, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T125106902945Z.pickle", + "timestamp_utc": "2026-08-01T12:51:07.021127Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T125106902945Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T125106902945Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03d078a644b884e98e587ff78823dd86fbfd084c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T125106902945Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5e9d94bd75a571b150a6bfeca1345cbe71188776a15bb5ba3963a5aa52c3a8c +size 5938144 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T125108195239Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T125108195239Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a5f7f83cb48a4f03615b1a7690797fb54f598a70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T125108195239Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113558571520, + "active_bytes_peak_delta": 23289421312, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113558571520, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113206249984, + "device": 6, + "inactive_split_bytes_current": 12819203072, + "inactive_split_bytes_peak": 13107691008, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 120342970368, + "reserved_bytes_peak": 120342970368, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T12:51:08.194820Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113206249984, + "allocated_bytes_peak_delta": 23631256576, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 90269150208, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 89574993408, + "device": 6, + "inactive_split_bytes_current": 8595538944, + "inactive_split_bytes_peak": 11273692160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 99067363328, + "reserved_bytes_peak": 99067363328, + "segment_count_current": 417, + "segment_count_peak": 417, + "timestamp_utc": "2026-08-01T12:51:07.040198Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12819203072, + "inactive_split_bytes_current_delta": 4223664128, + "inactive_split_bytes_peak": 13107691008, + "inactive_split_bytes_peak_delta": 1833998848, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 120342970368, + "reserved_bytes_current_delta": 21275607040, + "reserved_bytes_peak": 120342970368, + "reserved_bytes_peak_delta": 21275607040, + "segment_count_current": 486, + "segment_count_current_delta": 69, + "segment_count_peak": 486, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T125108195239Z.pickle", + "timestamp_utc": "2026-08-01T12:51:08.308116Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T125108195239Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T125108195239Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9dfe2cb876f3db4f9956e50dbff82e115fa3bc18 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T125108195239Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:860f59694873c5f9b4e5b89591e6972fa27ad3bd44a79c9e34b9246264f94d0e +size 5921155 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T125109469236Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T125109469236Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e155cdc2e1c0bbc4217e699dac3d66e267f462a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T125109469236Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113915239424, + "active_bytes_peak_delta": 356667904, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113915239424, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113210596352, + "device": 6, + "inactive_split_bytes_current": 11177133056, + "inactive_split_bytes_peak": 14272378368, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121167151104, + "reserved_bytes_peak": 121167151104, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:51:09.468812Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113210596352, + "allocated_bytes_peak_delta": 4346368, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113558571520, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113206249984, + "device": 6, + "inactive_split_bytes_current": 12819203072, + "inactive_split_bytes_peak": 13107691008, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 120342970368, + "reserved_bytes_peak": 120342970368, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T12:51:08.324542Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 11177133056, + "inactive_split_bytes_current_delta": -1642070016, + "inactive_split_bytes_peak": 14272378368, + "inactive_split_bytes_peak_delta": 1164687360, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121167151104, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 121167151104, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 487, + "segment_count_current_delta": 1, + "segment_count_peak": 487, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T125109469236Z.pickle", + "timestamp_utc": "2026-08-01T12:51:09.577766Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T125109469236Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T125109469236Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e503df5a40faa4a7fbeeb60e6d45559de7a1bad7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T125109469236Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9166045ebfcd8d3c7f2e5988320202dfac5a001ac68fb8c3c0ecec2b4980feb9 +size 6063611 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T125111299540Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T125111299540Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e0330bb750bb3616ca048c20249513da0ecfc676 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T125111299540Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 160876977152, + "active_bytes_current_delta": 88369356800, + "active_bytes_peak": 160930454528, + "active_bytes_peak_delta": 47015215104, + "after": { + "active_bytes_current": 160876977152, + "active_bytes_peak": 160930454528, + "allocated_bytes_current": 105026911232, + "allocated_bytes_peak": 113219151360, + "device": 6, + "inactive_split_bytes_current": 15730484224, + "inactive_split_bytes_peak": 22920594432, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 182911500288, + "reserved_bytes_peak": 182911500288, + "segment_count_current": 746, + "segment_count_peak": 746, + "timestamp_utc": "2026-08-01T12:51:11.298861Z", + "world_size": 8 + }, + "allocated_bytes_current": 105026911232, + "allocated_bytes_current_delta": 32519290880, + "allocated_bytes_peak": 113219151360, + "allocated_bytes_peak_delta": 8555008, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 113915239424, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 113210596352, + "device": 6, + "inactive_split_bytes_current": 11177133056, + "inactive_split_bytes_peak": 14272378368, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121167151104, + "reserved_bytes_peak": 121167151104, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T12:51:09.593595Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 15730484224, + "inactive_split_bytes_current_delta": 4553351168, + "inactive_split_bytes_peak": 22920594432, + "inactive_split_bytes_peak_delta": 8648216064, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 182911500288, + "reserved_bytes_current_delta": 61744349184, + "reserved_bytes_peak": 182911500288, + "reserved_bytes_peak_delta": 61744349184, + "segment_count_current": 746, + "segment_count_current_delta": 259, + "segment_count_peak": 746, + "segment_count_peak_delta": 259, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T125111299540Z.pickle", + "timestamp_utc": "2026-08-01T12:51:11.405687Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T125111299540Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T125111299540Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d1ab3e11223c05e9044917969b10f35a654bc88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T125111299540Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78d5a3eb4ca4a66d14e39c5e193f76dc799724b9c9d6c6712729709815046591 +size 5341984 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T125106902967Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T125106902967Z.json new file mode 100644 index 0000000000000000000000000000000000000000..40eac06c83acf5be2faf5730663b828aeb30efc7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T125106902967Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 37276165632, + "active_bytes_peak": 89929150464, + "active_bytes_peak_delta": 54462814208, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 89929150464, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 89576828928, + "device": 7, + "inactive_split_bytes_current": 8329200128, + "inactive_split_bytes_peak": 11725216768, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 98979282944, + "reserved_bytes_peak": 98979282944, + "segment_count_current": 418, + "segment_count_peak": 418, + "timestamp_utc": "2026-08-01T12:51:06.902409Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 37276165632, + "allocated_bytes_peak": 89576828928, + "allocated_bytes_peak_delta": 54110492672, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:49:58.746168Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8329200128, + "inactive_split_bytes_current_delta": 8196381184, + "inactive_split_bytes_peak": 11725216768, + "inactive_split_bytes_peak_delta": 11564830720, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 98979282944, + "reserved_bytes_current_delta": 59735277568, + "reserved_bytes_peak": 98979282944, + "reserved_bytes_peak_delta": 59735277568, + "segment_count_current": 418, + "segment_count_current_delta": 347, + "segment_count_peak": 418, + "segment_count_peak_delta": 347, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T125106902967Z.pickle", + "timestamp_utc": "2026-08-01T12:51:07.020122Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T125106902967Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T125106902967Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..390b85d3c9954946b383076b8ed811fdc6a37f75 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T125106902967Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fe81faf290a215c64036b94440d38c4cae9faa23a824f230c61927601fd25a6 +size 5957225 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T125108195328Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T125108195328Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e03fa2243a416560dd3364afe0ec1ca065db1c20 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T125108195328Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113908808704, + "active_bytes_peak_delta": 23979658240, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 113908808704, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 113204284416, + "device": 7, + "inactive_split_bytes_current": 11376361984, + "inactive_split_bytes_peak": 13157077504, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 120340873216, + "reserved_bytes_peak": 120340873216, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T12:51:08.194917Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113204284416, + "allocated_bytes_peak_delta": 23627455488, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 89929150464, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 89576828928, + "device": 7, + "inactive_split_bytes_current": 8329200128, + "inactive_split_bytes_peak": 11725216768, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 98979282944, + "reserved_bytes_peak": 98979282944, + "segment_count_current": 418, + "segment_count_peak": 418, + "timestamp_utc": "2026-08-01T12:51:07.037238Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11376361984, + "inactive_split_bytes_current_delta": 3047161856, + "inactive_split_bytes_peak": 13157077504, + "inactive_split_bytes_peak_delta": 1431860736, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 120340873216, + "reserved_bytes_current_delta": 21361590272, + "reserved_bytes_peak": 120340873216, + "reserved_bytes_peak_delta": 21361590272, + "segment_count_current": 485, + "segment_count_current_delta": 67, + "segment_count_peak": 485, + "segment_count_peak_delta": 67, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T125108195328Z.pickle", + "timestamp_utc": "2026-08-01T12:51:08.307053Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T125108195328Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T125108195328Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..516db0bfc1cb0a5bad2fee74fec0ee3f4b6caa68 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T125108195328Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74ca9b39b48b4af2d256994f78045dd72a66bdd0377f268ec35a3337fc335b25 +size 5923720 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T125109469236Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T125109469236Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f0fa1ff0f5fb8de1257cda6a2d71dec7b465e98 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T125109469236Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113912094208, + "active_bytes_peak_delta": 3285504, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 113912094208, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 113207451136, + "device": 7, + "inactive_split_bytes_current": 10908697088, + "inactive_split_bytes_peak": 13946454528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 121165053952, + "reserved_bytes_peak": 121165053952, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T12:51:09.468804Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113207451136, + "allocated_bytes_peak_delta": 3166720, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 113908808704, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 113204284416, + "device": 7, + "inactive_split_bytes_current": 11376361984, + "inactive_split_bytes_peak": 13157077504, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 120340873216, + "reserved_bytes_peak": 120340873216, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T12:51:08.321212Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10908697088, + "inactive_split_bytes_current_delta": -467664896, + "inactive_split_bytes_peak": 13946454528, + "inactive_split_bytes_peak_delta": 789377024, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 121165053952, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 121165053952, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 486, + "segment_count_current_delta": 1, + "segment_count_peak": 486, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T125109469236Z.pickle", + "timestamp_utc": "2026-08-01T12:51:09.577605Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T125109469236Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T125109469236Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..40e5de9d9854172531f4388e8fb6a2a29f8f04aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T125109469236Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:301362ca0a4b8fea306fdd6a51ec489630c240eb4a3f35766996495b85a3575f +size 6056312 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T125110950627Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T125110950627Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c164bbfb48cc65401a1573d26bd3f2ba5cd0e5e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T125110950627Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 161643664896, + "active_bytes_current_delta": 89136044032, + "active_bytes_peak": 161807195648, + "active_bytes_peak_delta": 47895101440, + "after": { + "active_bytes_current": 161643664896, + "active_bytes_peak": 161807195648, + "allocated_bytes_current": 105441277440, + "allocated_bytes_peak": 113221642240, + "device": 7, + "inactive_split_bytes_current": 15242717696, + "inactive_split_bytes_peak": 22737748480, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 182880043008, + "reserved_bytes_peak": 182880043008, + "segment_count_current": 747, + "segment_count_peak": 747, + "timestamp_utc": "2026-08-01T12:51:10.950048Z", + "world_size": 8 + }, + "allocated_bytes_current": 105441277440, + "allocated_bytes_current_delta": 32933656576, + "allocated_bytes_peak": 113221642240, + "allocated_bytes_peak_delta": 14191104, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 113912094208, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 113207451136, + "device": 7, + "inactive_split_bytes_current": 10908697088, + "inactive_split_bytes_peak": 13946454528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 121165053952, + "reserved_bytes_peak": 121165053952, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T12:51:09.591325Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 15242717696, + "inactive_split_bytes_current_delta": 4334020608, + "inactive_split_bytes_peak": 22737748480, + "inactive_split_bytes_peak_delta": 8791293952, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 182880043008, + "reserved_bytes_current_delta": 61714989056, + "reserved_bytes_peak": 182880043008, + "reserved_bytes_peak_delta": 61714989056, + "segment_count_current": 747, + "segment_count_current_delta": 261, + "segment_count_peak": 747, + "segment_count_peak_delta": 261, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T125110950627Z.pickle", + "timestamp_utc": "2026-08-01T12:51:11.053023Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T125110950627Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T125110950627Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..23c4641334da9d6082e784ca7ebd2a48d0841442 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T125110950627Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36f2f2b816d4fc8a5bef619a28fdafa5264d6439544a6cdde71bae15646295fa +size 5399402 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..45c7745721595e385acec491e5e67d25a077aa44 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 4096 +capacity: 1.50 +expert_dispatch_overlap: off +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6020 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 240 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T125328247379Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T125328247379Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a6cdb6edd2e186f96b860b4119dd03461b2f60c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T125328247379Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921626624, + "active_bytes_current_delta": 31690171392, + "active_bytes_peak": 92165365760, + "active_bytes_peak_delta": 56699029504, + "after": { + "active_bytes_current": 66921626624, + "active_bytes_peak": 92165365760, + "allocated_bytes_current": 66921626624, + "allocated_bytes_peak": 92165365760, + "device": 0, + "inactive_split_bytes_current": 7552435200, + "inactive_split_bytes_peak": 11778780160, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98754887680, + "reserved_bytes_peak": 98754887680, + "segment_count_current": 455, + "segment_count_peak": 455, + "timestamp_utc": "2026-08-01T12:53:28.246890Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921626624, + "allocated_bytes_current_delta": 31690171392, + "allocated_bytes_peak": 92165365760, + "allocated_bytes_peak_delta": 56699029504, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:52:18.302165Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7552435200, + "inactive_split_bytes_current_delta": 7417519104, + "inactive_split_bytes_peak": 11778780160, + "inactive_split_bytes_peak_delta": 11618148352, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98754887680, + "reserved_bytes_current_delta": 59508785152, + "reserved_bytes_peak": 98754887680, + "reserved_bytes_peak_delta": 59508785152, + "segment_count_current": 455, + "segment_count_current_delta": 383, + "segment_count_peak": 455, + "segment_count_peak_delta": 383, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T125328247379Z.pickle", + "timestamp_utc": "2026-08-01T12:53:28.356188Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T125328247379Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T125328247379Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eefdc112d98a503d7a60588d6fc8cc3bfd3287df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T125328247379Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f22e609b8d7692e1b8a1f3422b6c805fec8093857013df5e14f04d569f4995eb +size 5394901 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T125329660899Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T125329660899Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e1df2364ee7e0727a2e895a7be0cd2462aa7002a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T125329660899Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921626624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115795511296, + "active_bytes_peak_delta": 23630145536, + "after": { + "active_bytes_current": 66921626624, + "active_bytes_peak": 115795511296, + "allocated_bytes_current": 66921626624, + "allocated_bytes_peak": 115795511296, + "device": 0, + "inactive_split_bytes_current": 8913486848, + "inactive_split_bytes_peak": 11778780160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117329362944, + "reserved_bytes_peak": 117329362944, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T12:53:29.660491Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921626624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115795511296, + "allocated_bytes_peak_delta": 23630145536, + "before": { + "active_bytes_current": 66921626624, + "active_bytes_peak": 92165365760, + "allocated_bytes_current": 66921626624, + "allocated_bytes_peak": 92165365760, + "device": 0, + "inactive_split_bytes_current": 7552435200, + "inactive_split_bytes_peak": 11778780160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98754887680, + "reserved_bytes_peak": 98754887680, + "segment_count_current": 455, + "segment_count_peak": 455, + "timestamp_utc": "2026-08-01T12:53:28.372815Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8913486848, + "inactive_split_bytes_current_delta": 1361051648, + "inactive_split_bytes_peak": 11778780160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117329362944, + "reserved_bytes_current_delta": 18574475264, + "reserved_bytes_peak": 117329362944, + "reserved_bytes_peak_delta": 18574475264, + "segment_count_current": 505, + "segment_count_current_delta": 50, + "segment_count_peak": 505, + "segment_count_peak_delta": 50, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T125329660899Z.pickle", + "timestamp_utc": "2026-08-01T12:53:29.765668Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T125329660899Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T125329660899Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e68680e935a99ec7f79236232e6eb868b0950bd3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T125329660899Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e36321e361c53c6e3892bf637d9a13abb13cd61e0218e4a37f861eed3b82c6ca +size 5557297 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T125330823620Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T125330823620Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c8f7d2aa0fb66642bb621665068620498cd7323f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T125330823620Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922675200, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115802057216, + "active_bytes_peak_delta": 6545920, + "after": { + "active_bytes_current": 66922675200, + "active_bytes_peak": 115802057216, + "allocated_bytes_current": 66922675200, + "allocated_bytes_peak": 115802057216, + "device": 0, + "inactive_split_bytes_current": 7970817024, + "inactive_split_bytes_peak": 11778780160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118153543680, + "reserved_bytes_peak": 118153543680, + "segment_count_current": 506, + "segment_count_peak": 506, + "timestamp_utc": "2026-08-01T12:53:30.823215Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922675200, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115802057216, + "allocated_bytes_peak_delta": 6545920, + "before": { + "active_bytes_current": 66921626624, + "active_bytes_peak": 115795511296, + "allocated_bytes_current": 66921626624, + "allocated_bytes_peak": 115795511296, + "device": 0, + "inactive_split_bytes_current": 8913486848, + "inactive_split_bytes_peak": 11778780160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117329362944, + "reserved_bytes_peak": 117329362944, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T12:53:29.778668Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7970817024, + "inactive_split_bytes_current_delta": -942669824, + "inactive_split_bytes_peak": 11778780160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118153543680, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 118153543680, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 506, + "segment_count_current_delta": 1, + "segment_count_peak": 506, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T125330823620Z.pickle", + "timestamp_utc": "2026-08-01T12:53:30.923097Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T125330823620Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T125330823620Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..30bdb2db847751d20857e652719e88f183eb1191 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T125330823620Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9f49c628118f2f234f89f045902ef5011ac5c077511788adc48efa1dc4ef04f +size 5575763 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T125333499451Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T125333499451Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f9ab055d30c8e3d1ea8f716a3337fd557aec7fe6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T125333499451Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939587584, + "active_bytes_current_delta": 16912384, + "active_bytes_peak": 133961000960, + "active_bytes_peak_delta": 18158943744, + "after": { + "active_bytes_current": 66939587584, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66939587584, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_peak": 16775645184, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_peak": 149109604352, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:33.498902Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939587584, + "allocated_bytes_current_delta": 16912384, + "allocated_bytes_peak": 115807242240, + "allocated_bytes_peak_delta": 5185024, + "before": { + "active_bytes_current": 66922675200, + "active_bytes_peak": 115802057216, + "allocated_bytes_current": 66922675200, + "allocated_bytes_peak": 115802057216, + "device": 0, + "inactive_split_bytes_current": 7970817024, + "inactive_split_bytes_peak": 11778780160, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118153543680, + "reserved_bytes_peak": 118153543680, + "segment_count_current": 506, + "segment_count_peak": 506, + "timestamp_utc": "2026-08-01T12:53:30.934396Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_current_delta": 4166905856, + "inactive_split_bytes_peak": 16775645184, + "inactive_split_bytes_peak_delta": 4996865024, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_current_delta": 30956060672, + "reserved_bytes_peak": 149109604352, + "reserved_bytes_peak_delta": 30956060672, + "segment_count_current": 681, + "segment_count_current_delta": 175, + "segment_count_peak": 681, + "segment_count_peak_delta": 175, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T125333499451Z.pickle", + "timestamp_utc": "2026-08-01T12:53:33.619574Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T125333499451Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T125333499451Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a7c26e306e9bd62c18869864b13edfbebd51d93 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T125333499451Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:434d1f66c962d664b34e836df52270e73e5061b6b3dee117fc07065dd34a6f9e +size 5831738 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T125334599481Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T125334599481Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b5966895fbde1cb5cc4c32b03a8df58ec4574ba3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T125334599481Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939587584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133961000960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939587584, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66939587584, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_peak": 16775645184, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_peak": 149109604352, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:34.599034Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939587584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807242240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939587584, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66939587584, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_peak": 16775645184, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_peak": 149109604352, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:33.632480Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16775645184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149109604352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 681, + "segment_count_current_delta": 0, + "segment_count_peak": 681, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T125334599481Z.pickle", + "timestamp_utc": "2026-08-01T12:53:34.605407Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T125334599481Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T125334599481Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..53861f5f8ac153d7a706beb597fbb17dcbefa958 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T125334599481Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83012e61b4c5746914d8711b3c34b559391ee2819a30c3f9186cdc040452cff8 +size 208360 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T125335555942Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T125335555942Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1df04b0b4cec30b403c97436ca1187ac62e164f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T125335555942Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939587584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133961000960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939587584, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66939587584, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_peak": 16775645184, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_peak": 149109604352, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:35.555543Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939587584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807242240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939587584, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66939587584, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_peak": 16775645184, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_peak": 149109604352, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:34.606341Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16775645184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149109604352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 681, + "segment_count_current_delta": 0, + "segment_count_peak": 681, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T125335555942Z.pickle", + "timestamp_utc": "2026-08-01T12:53:35.561919Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T125335555942Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T125335555942Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1efcea57b2570d4d7ca477a4f007c3123a1b6227 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T125335555942Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a32bdbe8de79e427778524e4a137bc249b63cb6a8effdf8ceab346e668b81a8 +size 208360 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T125336519506Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T125336519506Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c2263bdcb4ece4fe43aa07253d4bda08d52ab770 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T125336519506Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939587584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133961000960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939587584, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66939587584, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_peak": 16775645184, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_peak": 149109604352, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:36.519085Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939587584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807242240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939587584, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66939587584, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_peak": 16775645184, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_peak": 149109604352, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:35.562842Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16775645184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149109604352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 681, + "segment_count_current_delta": 0, + "segment_count_peak": 681, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T125336519506Z.pickle", + "timestamp_utc": "2026-08-01T12:53:36.525446Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T125336519506Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T125336519506Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0383f16e158c18c430f48ce72575e653f88450a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T125336519506Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f04c03feb947fa502e4b2dd8340d039c4afde29fe159cb740e230fc0dae098a3 +size 208360 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T125337473902Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T125337473902Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6707b12dfa8559eb9a6a704ef5fc344640e54480 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T125337473902Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939587584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133961000960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939587584, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66939587584, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_peak": 16775645184, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_peak": 149109604352, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:37.473504Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939587584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807242240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939587584, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66939587584, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_peak": 16775645184, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_peak": 149109604352, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:36.526347Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16775645184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149109604352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 681, + "segment_count_current_delta": 0, + "segment_count_peak": 681, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T125337473902Z.pickle", + "timestamp_utc": "2026-08-01T12:53:37.479854Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T125337473902Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T125337473902Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d32128524a932c68ad5a848f39970c53cfc47d98 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T125337473902Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:369a7b0e3610dc23bc6ba75fa7ca953eac34704acb95c2d17b66648b0f91c082 +size 208360 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T125338426762Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T125338426762Z.json new file mode 100644 index 0000000000000000000000000000000000000000..422049c0fef1bb8261569689c33d72771202cae3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T125338426762Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939587584, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133961000960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939587584, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66939587584, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_peak": 16775645184, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_peak": 149109604352, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:38.426351Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939587584, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807242240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939587584, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66939587584, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_peak": 16775645184, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_peak": 149109604352, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:37.480780Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16775645184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149109604352, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 681, + "segment_count_current_delta": 0, + "segment_count_peak": 681, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T125338426762Z.pickle", + "timestamp_utc": "2026-08-01T12:53:38.432885Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T125338426762Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T125338426762Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c91d8dd59ee3d9f0cfc4c8fe2af5e2ceb40a5933 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T125338426762Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e10a129b6359edca49fa4eb2d9a4317c90db502350565a261f74fcdaf0feccb +size 208360 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T125437081038Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T125437081038Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3778388df8384a3105b516b65f3618f2629bb672 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T125437081038Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66956514304, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 133961000960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66956514304, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66956514304, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 5240040448, + "inactive_split_bytes_peak": 16775645184, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150002991104, + "reserved_bytes_peak": 150002991104, + "segment_count_current": 687, + "segment_count_peak": 687, + "timestamp_utc": "2026-08-01T12:54:37.080587Z", + "world_size": 8 + }, + "allocated_bytes_current": 66956514304, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 115807242240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939587584, + "active_bytes_peak": 133961000960, + "allocated_bytes_current": 66939587584, + "allocated_bytes_peak": 115807242240, + "device": 0, + "inactive_split_bytes_current": 12137722880, + "inactive_split_bytes_peak": 16775645184, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 149109604352, + "reserved_bytes_peak": 149109604352, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:38.433787Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5240040448, + "inactive_split_bytes_current_delta": -6897682432, + "inactive_split_bytes_peak": 16775645184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150002991104, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150002991104, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 687, + "segment_count_current_delta": 6, + "segment_count_peak": 687, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T125437081038Z.pickle", + "timestamp_utc": "2026-08-01T12:54:37.163747Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T125437081038Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T125437081038Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7cc0b7203ced4b8d88172e97b7a18a56f7111cf6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T125437081038Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d7be1ebbb33b7b946c745b223113419598e984727dc5635aa8141658584d371 +size 4026242 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T125328247398Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T125328247398Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ab9e654dfd701ea93987a31f51f4fbf952e7da0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T125328247398Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 92165955584, + "active_bytes_peak_delta": 56699619328, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92165955584, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92165955584, + "device": 1, + "inactive_split_bytes_current": 6842008576, + "inactive_split_bytes_peak": 12074767360, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 97813266432, + "reserved_bytes_peak": 97813266432, + "segment_count_current": 452, + "segment_count_peak": 452, + "timestamp_utc": "2026-08-01T12:53:28.246917Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 92165955584, + "allocated_bytes_peak_delta": 56699619328, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:52:18.301948Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6842008576, + "inactive_split_bytes_current_delta": 6709189632, + "inactive_split_bytes_peak": 12074767360, + "inactive_split_bytes_peak_delta": 11914381312, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 97813266432, + "reserved_bytes_current_delta": 58569261056, + "reserved_bytes_peak": 97813266432, + "reserved_bytes_peak_delta": 58569261056, + "segment_count_current": 452, + "segment_count_current_delta": 381, + "segment_count_peak": 452, + "segment_count_peak_delta": 381, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T125328247398Z.pickle", + "timestamp_utc": "2026-08-01T12:53:28.361799Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T125328247398Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T125328247398Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d8b702b269ee93759fc001ff76385850d5c2f600 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T125328247398Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47d637810fac6c3bdf508fb7bcb7a64af2170d5ee8d2ba24d0b220bdeae9da37 +size 5400440 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T125329660962Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T125329660962Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9adfb72dcc9ec22986e884e2a2bd65b9b781d181 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T125329660962Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115806492672, + "active_bytes_peak_delta": 23640537088, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115806492672, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115806492672, + "device": 1, + "inactive_split_bytes_current": 8198865920, + "inactive_split_bytes_peak": 12074767360, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118088531968, + "reserved_bytes_peak": 118088531968, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:53:29.660549Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806492672, + "allocated_bytes_peak_delta": 23640537088, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92165955584, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92165955584, + "device": 1, + "inactive_split_bytes_current": 6842008576, + "inactive_split_bytes_peak": 12074767360, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 97813266432, + "reserved_bytes_peak": 97813266432, + "segment_count_current": 452, + "segment_count_peak": 452, + "timestamp_utc": "2026-08-01T12:53:28.381464Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8198865920, + "inactive_split_bytes_current_delta": 1356857344, + "inactive_split_bytes_peak": 12074767360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118088531968, + "reserved_bytes_current_delta": 20275265536, + "reserved_bytes_peak": 118088531968, + "reserved_bytes_peak_delta": 20275265536, + "segment_count_current": 503, + "segment_count_current_delta": 51, + "segment_count_peak": 503, + "segment_count_peak_delta": 51, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T125329660962Z.pickle", + "timestamp_utc": "2026-08-01T12:53:29.769651Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T125329660962Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T125329660962Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1b038deff75809f5df478ca33965ce656d1d07d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T125329660962Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9ab3578fc7fd3980b71d81b3d1116f3fef13dc0c4336388613a1b3352385097 +size 5535620 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T125330823670Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T125330823670Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5158794c821e4f9fbc321639866c46b4b8bd229c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T125330823670Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115808043008, + "active_bytes_peak_delta": 1550336, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115808043008, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115808043008, + "device": 1, + "inactive_split_bytes_current": 7789921280, + "inactive_split_bytes_peak": 12074767360, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118088531968, + "reserved_bytes_peak": 118088531968, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:53:30.823256Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115808043008, + "allocated_bytes_peak_delta": 1550336, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115806492672, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115806492672, + "device": 1, + "inactive_split_bytes_current": 8198865920, + "inactive_split_bytes_peak": 12074767360, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118088531968, + "reserved_bytes_peak": 118088531968, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:53:29.785175Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7789921280, + "inactive_split_bytes_current_delta": -408944640, + "inactive_split_bytes_peak": 12074767360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118088531968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118088531968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T125330823670Z.pickle", + "timestamp_utc": "2026-08-01T12:53:30.929528Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T125330823670Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T125330823670Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..125f9b5f3965c2d9908046784f7860303e8adc00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T125330823670Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eeec8bb38b8d8b22f2f20b65051509738ea6bf3e650b96a2558d44f365fac848 +size 5569211 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T125333499562Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T125333499562Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94b6a841cd1ab9cf7340382727132649da674b57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T125333499562Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 133966784512, + "active_bytes_peak_delta": 18158741504, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_peak": 17060003328, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149220753408, + "reserved_bytes_peak": 149220753408, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:33.499031Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 115813025792, + "allocated_bytes_peak_delta": 4982784, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115808043008, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115808043008, + "device": 1, + "inactive_split_bytes_current": 7789921280, + "inactive_split_bytes_peak": 12074767360, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118088531968, + "reserved_bytes_peak": 118088531968, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:53:30.950289Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_current_delta": 4362989568, + "inactive_split_bytes_peak": 17060003328, + "inactive_split_bytes_peak_delta": 4985235968, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149220753408, + "reserved_bytes_current_delta": 31132221440, + "reserved_bytes_peak": 149220753408, + "reserved_bytes_peak_delta": 31132221440, + "segment_count_current": 681, + "segment_count_current_delta": 178, + "segment_count_peak": 681, + "segment_count_peak_delta": 178, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T125333499562Z.pickle", + "timestamp_utc": "2026-08-01T12:53:33.625972Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T125333499562Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T125333499562Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a93f1a68a5d93d170780aab8124533bcd17e2b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T125333499562Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a27afcfa58e2191ab2fde1fb459fb6678a3f11b33356b02e6653b55e77e339cc +size 5830307 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T125334599673Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T125334599673Z.json new file mode 100644 index 0000000000000000000000000000000000000000..344989f1a09f4eb5c3eb5f8a254ef66bc47fe5f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T125334599673Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133966784512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_peak": 17060003328, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149220753408, + "reserved_bytes_peak": 149220753408, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:34.599108Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115813025792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_peak": 17060003328, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149220753408, + "reserved_bytes_peak": 149220753408, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:33.645052Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17060003328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149220753408, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149220753408, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 681, + "segment_count_current_delta": 0, + "segment_count_peak": 681, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T125334599673Z.pickle", + "timestamp_utc": "2026-08-01T12:53:34.606003Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T125334599673Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T125334599673Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b7d0d886bcc59efa2a0738b38dab3690dcf54b66 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T125334599673Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f73e2ba09da6f7a7b7d87e8deaac0a0a5bf6d809bbfd20dfb92068ce4dd6d803 +size 205408 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T125335555997Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T125335555997Z.json new file mode 100644 index 0000000000000000000000000000000000000000..921fdbe1518f23b5399e3cb59a00d8df8bb969c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T125335555997Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133966784512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_peak": 17060003328, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149220753408, + "reserved_bytes_peak": 149220753408, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:35.555578Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115813025792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_peak": 17060003328, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149220753408, + "reserved_bytes_peak": 149220753408, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:34.606959Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17060003328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149220753408, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149220753408, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 681, + "segment_count_current_delta": 0, + "segment_count_peak": 681, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T125335555997Z.pickle", + "timestamp_utc": "2026-08-01T12:53:35.561790Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T125335555997Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T125335555997Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ea3ce0fdd3740064bbf379e76b318cc78a4767b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T125335555997Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c4d48d11fb1f06e746cb9a83b195e95018d6af2c2980d07aa6d52f0feea9484 +size 205408 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T125336519539Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T125336519539Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d0e0f8d9d3fa2567bfc31f33e5a5dde376fd36d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T125336519539Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133966784512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_peak": 17060003328, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149220753408, + "reserved_bytes_peak": 149220753408, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:36.519122Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115813025792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_peak": 17060003328, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149220753408, + "reserved_bytes_peak": 149220753408, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:35.562700Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17060003328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149220753408, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149220753408, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 681, + "segment_count_current_delta": 0, + "segment_count_peak": 681, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T125336519539Z.pickle", + "timestamp_utc": "2026-08-01T12:53:36.525345Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T125336519539Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T125336519539Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..488ce656484b2d24da33f9a1f07c51e82310944e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T125336519539Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6addf671d385b4eabb70a3fdc426edb45bb407c607de854ae14808b779a33fde +size 205408 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T125337474001Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T125337474001Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2f144afc96f19dc429e75c453c158862dd2fa17d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T125337474001Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133966784512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_peak": 17060003328, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149220753408, + "reserved_bytes_peak": 149220753408, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:37.473580Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115813025792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_peak": 17060003328, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149220753408, + "reserved_bytes_peak": 149220753408, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:36.526366Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17060003328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149220753408, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149220753408, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 681, + "segment_count_current_delta": 0, + "segment_count_peak": 681, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T125337474001Z.pickle", + "timestamp_utc": "2026-08-01T12:53:37.479793Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T125337474001Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T125337474001Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c22fe94d3b24b542fa65e97af1d183b991a3c292 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T125337474001Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84d3e8c0298ef0e614399251a2ab85751dad7a3e935bc9f0a8a105179f63f392 +size 205408 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T125338426778Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T125338426778Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4346612fee416ab6a9b769a8ca7ce4f829d0d86a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T125338426778Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133966784512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_peak": 17060003328, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149220753408, + "reserved_bytes_peak": 149220753408, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:38.426384Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115813025792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_peak": 17060003328, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149220753408, + "reserved_bytes_peak": 149220753408, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:37.480726Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17060003328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149220753408, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149220753408, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 681, + "segment_count_current_delta": 0, + "segment_count_peak": 681, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T125338426778Z.pickle", + "timestamp_utc": "2026-08-01T12:53:38.432373Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T125338426778Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T125338426778Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb7a379f99f134acee80aa7eae178182ade72e58 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T125338426778Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17da72708b8a1f7b0bab45d9e3ec07767e8ab8ed8891d948b5a8bb459f7e19e2 +size 205408 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T125437081155Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T125437081155Z.json new file mode 100644 index 0000000000000000000000000000000000000000..49181844ede66dcc9be4303233b899e4d2b588cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T125437081155Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66956006400, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 133966784512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66956006400, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66956006400, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 5018250240, + "inactive_split_bytes_peak": 17060003328, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 149289959424, + "reserved_bytes_peak": 149289959424, + "segment_count_current": 686, + "segment_count_peak": 686, + "timestamp_utc": "2026-08-01T12:54:37.080631Z", + "world_size": 8 + }, + "allocated_bytes_current": 66956006400, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 115813025792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133966784512, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115813025792, + "device": 1, + "inactive_split_bytes_current": 12152910848, + "inactive_split_bytes_peak": 17060003328, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 149220753408, + "reserved_bytes_peak": 149220753408, + "segment_count_current": 681, + "segment_count_peak": 681, + "timestamp_utc": "2026-08-01T12:53:38.433292Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5018250240, + "inactive_split_bytes_current_delta": -7134660608, + "inactive_split_bytes_peak": 17060003328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 149289959424, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 149289959424, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 686, + "segment_count_current_delta": 5, + "segment_count_peak": 686, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T125437081155Z.pickle", + "timestamp_utc": "2026-08-01T12:54:37.168064Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T125437081155Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T125437081155Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b9bbafdfcdf23da1019e07e5080f7e69371132c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T125437081155Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a617bd1c3e973e4a99084c274d5e9734048afe97c8758547eb2664a053f0ec40 +size 4068414 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T125328247316Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T125328247316Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ac56861f482104e8da1f3a2c653a0886e0d3aa9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T125328247316Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 92161499136, + "active_bytes_peak_delta": 56695162880, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92161499136, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92161499136, + "device": 2, + "inactive_split_bytes_current": 7318062080, + "inactive_split_bytes_peak": 10965488640, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98750693376, + "reserved_bytes_peak": 98750693376, + "segment_count_current": 453, + "segment_count_peak": 453, + "timestamp_utc": "2026-08-01T12:53:28.246845Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 92161499136, + "allocated_bytes_peak_delta": 56695162880, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:52:18.302106Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7318062080, + "inactive_split_bytes_current_delta": 7185243136, + "inactive_split_bytes_peak": 10965488640, + "inactive_split_bytes_peak_delta": 10805102592, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98750693376, + "reserved_bytes_current_delta": 59506688000, + "reserved_bytes_peak": 98750693376, + "reserved_bytes_peak_delta": 59506688000, + "segment_count_current": 453, + "segment_count_current_delta": 382, + "segment_count_peak": 453, + "segment_count_peak_delta": 382, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T125328247316Z.pickle", + "timestamp_utc": "2026-08-01T12:53:28.358566Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T125328247316Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T125328247316Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8314a9a6be56d74d31907c404ee8e5748a1ee8b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T125328247316Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:883c2d1352e5f065892f2a6e844e678aaaa06e4bae6f3c2485f078b78bd6833f +size 5385943 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T125329660919Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T125329660919Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6491e0bc944a27f35fc177df7abc337440926efd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T125329660919Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115805884928, + "active_bytes_peak_delta": 23644385792, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115805884928, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115805884928, + "device": 2, + "inactive_split_bytes_current": 8431649792, + "inactive_split_bytes_peak": 10965488640, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:29.660502Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115805884928, + "allocated_bytes_peak_delta": 23644385792, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92161499136, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92161499136, + "device": 2, + "inactive_split_bytes_current": 7318062080, + "inactive_split_bytes_peak": 10965488640, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98750693376, + "reserved_bytes_peak": 98750693376, + "segment_count_current": 453, + "segment_count_peak": 453, + "timestamp_utc": "2026-08-01T12:53:28.375438Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8431649792, + "inactive_split_bytes_current_delta": 1113587712, + "inactive_split_bytes_peak": 10965488640, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118086434816, + "reserved_bytes_current_delta": 19335741440, + "reserved_bytes_peak": 118086434816, + "reserved_bytes_peak_delta": 19335741440, + "segment_count_current": 499, + "segment_count_current_delta": 46, + "segment_count_peak": 499, + "segment_count_peak_delta": 46, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T125329660919Z.pickle", + "timestamp_utc": "2026-08-01T12:53:29.768924Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T125329660919Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T125329660919Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..af571978440f912846f165282e0e95c11e8408df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T125329660919Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95c92ddc6bb3a1d813e6214ad2f137bf38621eb8c10918632bf34c2fd782958c +size 5567931 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T125330823628Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T125330823628Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ceb44e18e192c01fce4390394cc5dc1059cafc1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T125330823628Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115808364544, + "active_bytes_peak_delta": 2479616, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115808364544, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115808364544, + "device": 2, + "inactive_split_bytes_current": 7911556096, + "inactive_split_bytes_peak": 10965488640, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:30.823223Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115808364544, + "allocated_bytes_peak_delta": 2479616, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115805884928, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115805884928, + "device": 2, + "inactive_split_bytes_current": 8431649792, + "inactive_split_bytes_peak": 10965488640, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:29.784448Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7911556096, + "inactive_split_bytes_current_delta": -520093696, + "inactive_split_bytes_peak": 10965488640, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118086434816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118086434816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T125330823628Z.pickle", + "timestamp_utc": "2026-08-01T12:53:30.924906Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T125330823628Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T125330823628Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1825f3a04543396350fbb31e70efff7fc831c995 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T125330823628Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2f18f41d76866cfa0a89bd28db0c96f3651ee39d0a5f2e050cac49d075cfd9e +size 5567776 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T125333499519Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T125333499519Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb0cb8aa68f4b8f37c95cf8400ee6766d0368659 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T125333499519Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 133968619520, + "active_bytes_peak_delta": 18160254976, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_peak": 16320628736, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 148780351488, + "reserved_bytes_peak": 148780351488, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:53:33.498985Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115814860800, + "allocated_bytes_peak_delta": 6496256, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115808364544, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115808364544, + "device": 2, + "inactive_split_bytes_current": 7911556096, + "inactive_split_bytes_peak": 10965488640, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:30.937575Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_current_delta": 4345032704, + "inactive_split_bytes_peak": 16320628736, + "inactive_split_bytes_peak_delta": 5355140096, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 148780351488, + "reserved_bytes_current_delta": 30693916672, + "reserved_bytes_peak": 148780351488, + "reserved_bytes_peak_delta": 30693916672, + "segment_count_current": 683, + "segment_count_current_delta": 184, + "segment_count_peak": 683, + "segment_count_peak_delta": 184, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T125333499519Z.pickle", + "timestamp_utc": "2026-08-01T12:53:33.623447Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T125333499519Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T125333499519Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..606f72a7f607f4be9f43bed8f4f693a8b57b83f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T125333499519Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a7afa17b343520373e18fedf04c04e6ffbf38754ca578219c654709f2784ad0 +size 5826843 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T125334599525Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T125334599525Z.json new file mode 100644 index 0000000000000000000000000000000000000000..21fa7ff5726c5279ae841f863f880f1e0711bed1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T125334599525Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133968619520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_peak": 16320628736, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 148780351488, + "reserved_bytes_peak": 148780351488, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:53:34.599057Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115814860800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_peak": 16320628736, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 148780351488, + "reserved_bytes_peak": 148780351488, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:53:33.640491Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16320628736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 148780351488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148780351488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 683, + "segment_count_current_delta": 0, + "segment_count_peak": 683, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T125334599525Z.pickle", + "timestamp_utc": "2026-08-01T12:53:34.605513Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T125334599525Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T125334599525Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c2411feeb642bf0833ca14d16eb050e1fb3cb96a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T125334599525Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc6c237a6c21d6e7f05bbd41323a2568fb0143f9ece7a3dbdaa9b48a154fc794 +size 205693 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T125335555922Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T125335555922Z.json new file mode 100644 index 0000000000000000000000000000000000000000..54682a166b83a6cb96cee8971d774f183f68a604 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T125335555922Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133968619520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_peak": 16320628736, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 148780351488, + "reserved_bytes_peak": 148780351488, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:53:35.555525Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115814860800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_peak": 16320628736, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 148780351488, + "reserved_bytes_peak": 148780351488, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:53:34.606441Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16320628736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 148780351488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148780351488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 683, + "segment_count_current_delta": 0, + "segment_count_peak": 683, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T125335555922Z.pickle", + "timestamp_utc": "2026-08-01T12:53:35.561506Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T125335555922Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T125335555922Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cbada0a3fac3dccaa27320eda1be5ebf9dc9941e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T125335555922Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0488316efe9674a92b31441cb74777610a48365626a7962899771734fc5a3ffe +size 205693 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T125336521114Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T125336521114Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c780d8f94c814c31fb14f0607f23d17a0a4a4a19 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T125336521114Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133968619520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_peak": 16320628736, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 148780351488, + "reserved_bytes_peak": 148780351488, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:53:36.519807Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115814860800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_peak": 16320628736, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 148780351488, + "reserved_bytes_peak": 148780351488, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:53:35.562409Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16320628736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 148780351488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148780351488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 683, + "segment_count_current_delta": 0, + "segment_count_peak": 683, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T125336521114Z.pickle", + "timestamp_utc": "2026-08-01T12:53:36.526978Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T125336521114Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T125336521114Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7c2ba465568050ae87c1e423730599ee200208ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T125336521114Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4029d7a6b2177a1ae5604e012f6ac23905097ffe3a18d53769f20e4398e89669 +size 205693 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T125337473922Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T125337473922Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e080bd3a409f54cdf4356ed09e71b1d957d849a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T125337473922Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133968619520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_peak": 16320628736, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 148780351488, + "reserved_bytes_peak": 148780351488, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:53:37.473518Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115814860800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_peak": 16320628736, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 148780351488, + "reserved_bytes_peak": 148780351488, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:53:36.527886Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16320628736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 148780351488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148780351488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 683, + "segment_count_current_delta": 0, + "segment_count_peak": 683, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T125337473922Z.pickle", + "timestamp_utc": "2026-08-01T12:53:37.479516Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T125337473922Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T125337473922Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4cd2b6442e0b2064aa87a22afcb553ad23c9effa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T125337473922Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c9f75b9916bfc1d531bd25a7c47ee611a16a7e2edc3d5e8235555824c886e19 +size 205693 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T125338426765Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T125338426765Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5208a4aa8a0000a777718ca6eae439124a1b49a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T125338426765Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133968619520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_peak": 16320628736, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 148780351488, + "reserved_bytes_peak": 148780351488, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:53:38.426345Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115814860800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_peak": 16320628736, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 148780351488, + "reserved_bytes_peak": 148780351488, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:53:37.480444Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16320628736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 148780351488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148780351488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 683, + "segment_count_current_delta": 0, + "segment_count_peak": 683, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T125338426765Z.pickle", + "timestamp_utc": "2026-08-01T12:53:38.432499Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T125338426765Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T125338426765Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a243f8c1f2bad5c2e37ac32e3fb21c9eb3879be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T125338426765Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e31ee0cfe62ddf7f48ae4fb4ea24d07aecb379d190b8b73533bedb8f7a66aad +size 205693 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T125437081153Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T125437081153Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27aef3067a1d855f650cb63d0e6ef9134df3f8ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T125437081153Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955088896, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 133968619520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955088896, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66955088896, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 4006243328, + "inactive_split_bytes_peak": 16320628736, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 149673738240, + "reserved_bytes_peak": 149673738240, + "segment_count_current": 689, + "segment_count_peak": 689, + "timestamp_utc": "2026-08-01T12:54:37.080657Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955088896, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 115814860800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 133968619520, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115814860800, + "device": 2, + "inactive_split_bytes_current": 12256588800, + "inactive_split_bytes_peak": 16320628736, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 148780351488, + "reserved_bytes_peak": 148780351488, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:53:38.435972Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4006243328, + "inactive_split_bytes_current_delta": -8250345472, + "inactive_split_bytes_peak": 16320628736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 149673738240, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 149673738240, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 689, + "segment_count_current_delta": 6, + "segment_count_peak": 689, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T125437081153Z.pickle", + "timestamp_utc": "2026-08-01T12:54:37.169970Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T125437081153Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T125437081153Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7cb63a4bb54242b129090d5f80a1e53e32694bf1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T125437081153Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76e22bc038a2398cba566769df2ca324d6ba0e42f85060c469f698d61436829b +size 4003340 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T125328247427Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T125328247427Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c15b328539555623a698f44142ab5f3db67c86ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T125328247427Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 92165955584, + "active_bytes_peak_delta": 56699619328, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92165955584, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92165955584, + "device": 3, + "inactive_split_bytes_current": 6842008576, + "inactive_split_bytes_peak": 12072670208, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 97811169280, + "reserved_bytes_peak": 97811169280, + "segment_count_current": 451, + "segment_count_peak": 451, + "timestamp_utc": "2026-08-01T12:53:28.246909Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 92165955584, + "allocated_bytes_peak_delta": 56699619328, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:52:18.302226Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6842008576, + "inactive_split_bytes_current_delta": 6709189632, + "inactive_split_bytes_peak": 12072670208, + "inactive_split_bytes_peak_delta": 11912284160, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 97811169280, + "reserved_bytes_current_delta": 58567163904, + "reserved_bytes_peak": 97811169280, + "reserved_bytes_peak_delta": 58567163904, + "segment_count_current": 451, + "segment_count_current_delta": 380, + "segment_count_peak": 451, + "segment_count_peak_delta": 380, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T125328247427Z.pickle", + "timestamp_utc": "2026-08-01T12:53:28.362317Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T125328247427Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T125328247427Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f088257bc2bf27b90fefe0045fbb6abc71bb7f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T125328247427Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20a4651a05422db4c2623c00d47d225c224fb2b3428cdbcfd581e9f9d9d98726 +size 5546631 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T125329660946Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T125329660946Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f7fee06a0cadf41d28ce7b78212f9516acfecf5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T125329660946Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115806492672, + "active_bytes_peak_delta": 23640537088, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115806492672, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115806492672, + "device": 3, + "inactive_split_bytes_current": 8200963072, + "inactive_split_bytes_peak": 12072670208, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118088531968, + "reserved_bytes_peak": 118088531968, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:53:29.660511Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806492672, + "allocated_bytes_peak_delta": 23640537088, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92165955584, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92165955584, + "device": 3, + "inactive_split_bytes_current": 6842008576, + "inactive_split_bytes_peak": 12072670208, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 97811169280, + "reserved_bytes_peak": 97811169280, + "segment_count_current": 451, + "segment_count_peak": 451, + "timestamp_utc": "2026-08-01T12:53:28.381326Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8200963072, + "inactive_split_bytes_current_delta": 1358954496, + "inactive_split_bytes_peak": 12072670208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118088531968, + "reserved_bytes_current_delta": 20277362688, + "reserved_bytes_peak": 118088531968, + "reserved_bytes_peak_delta": 20277362688, + "segment_count_current": 503, + "segment_count_current_delta": 52, + "segment_count_peak": 503, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T125329660946Z.pickle", + "timestamp_utc": "2026-08-01T12:53:29.769466Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T125329660946Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T125329660946Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..52892115bff52275837ab89ed48a747b281bf391 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T125329660946Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3125dc20d1a6f23255ba379085659d0761bf8d9f858ce006d6abd83514a84055 +size 5537140 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T125330823688Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T125330823688Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e8736f4516b8c9ac6d4a05127066fa2494972577 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T125330823688Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115808043008, + "active_bytes_peak_delta": 1550336, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115808043008, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115808043008, + "device": 3, + "inactive_split_bytes_current": 7785726976, + "inactive_split_bytes_peak": 12072670208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118088531968, + "reserved_bytes_peak": 118088531968, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:53:30.823261Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115808043008, + "allocated_bytes_peak_delta": 1550336, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115806492672, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115806492672, + "device": 3, + "inactive_split_bytes_current": 8200963072, + "inactive_split_bytes_peak": 12072670208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118088531968, + "reserved_bytes_peak": 118088531968, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:53:29.785186Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7785726976, + "inactive_split_bytes_current_delta": -415236096, + "inactive_split_bytes_peak": 12072670208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118088531968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118088531968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T125330823688Z.pickle", + "timestamp_utc": "2026-08-01T12:53:30.927537Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T125330823688Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T125330823688Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fc34abccbdf1604bf64700e58ef518a3d20ec5d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T125330823688Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19f948904d7d95547e95013f392a9490517b745e535dfeb96619e73e1664030c +size 5568770 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T125333499496Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T125333499496Z.json new file mode 100644 index 0000000000000000000000000000000000000000..466d9179cf9dbdff1607d76712b71b2812636bc7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T125333499496Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 133963638784, + "active_bytes_peak_delta": 18155595776, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_peak": 17777560064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149275279360, + "reserved_bytes_peak": 149275279360, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:33.498933Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 115809880064, + "allocated_bytes_peak_delta": 1837056, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115808043008, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115808043008, + "device": 3, + "inactive_split_bytes_current": 7785726976, + "inactive_split_bytes_peak": 12072670208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118088531968, + "reserved_bytes_peak": 118088531968, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T12:53:30.942476Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_current_delta": 4549636096, + "inactive_split_bytes_peak": 17777560064, + "inactive_split_bytes_peak_delta": 5704889856, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149275279360, + "reserved_bytes_current_delta": 31186747392, + "reserved_bytes_peak": 149275279360, + "reserved_bytes_peak_delta": 31186747392, + "segment_count_current": 679, + "segment_count_current_delta": 176, + "segment_count_peak": 679, + "segment_count_peak_delta": 176, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T125333499496Z.pickle", + "timestamp_utc": "2026-08-01T12:53:33.618128Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T125333499496Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T125333499496Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..33bd78c53fa9a7b9fc97131f4f8878db13cff67a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T125333499496Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6971e67df64c71dde5615a5f774064807641320de88bc52aaf496c10b275aba +size 5796370 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T125334599579Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T125334599579Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6a1be5d733e10be15b2266b586f2d4fd55d3e6c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T125334599579Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133963638784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_peak": 17777560064, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149275279360, + "reserved_bytes_peak": 149275279360, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:34.599098Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115809880064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_peak": 17777560064, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149275279360, + "reserved_bytes_peak": 149275279360, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:33.634501Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17777560064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149275279360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149275279360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T125334599579Z.pickle", + "timestamp_utc": "2026-08-01T12:53:34.606044Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T125334599579Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T125334599579Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..83352b8aaf9ef88593f5978e93dbd73b5409a795 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T125334599579Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce9a28112af62234eeb74db0e64d24ca63a436cad0ac9d7efd0b103c59f6f7a1 +size 203826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T125335555995Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T125335555995Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e42ed964f872dc5be4f86f09d37741658503711 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T125335555995Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133963638784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_peak": 17777560064, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149275279360, + "reserved_bytes_peak": 149275279360, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:35.555585Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115809880064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_peak": 17777560064, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149275279360, + "reserved_bytes_peak": 149275279360, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:34.607023Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17777560064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149275279360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149275279360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T125335555995Z.pickle", + "timestamp_utc": "2026-08-01T12:53:35.561795Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T125335555995Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T125335555995Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aa458843223ed66174deb2368ffde2114c68fc29 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T125335555995Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:913ed9bfadc0998b44a85e31ad10a28ec56bba1cb9bdf1f375da45ee9012fe92 +size 203826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T125336519790Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T125336519790Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80c3642a638699b6c1bbf1dd16207fbd68689ea1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T125336519790Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133963638784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_peak": 17777560064, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149275279360, + "reserved_bytes_peak": 149275279360, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:36.519339Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115809880064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_peak": 17777560064, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149275279360, + "reserved_bytes_peak": 149275279360, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:35.562736Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17777560064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149275279360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149275279360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T125336519790Z.pickle", + "timestamp_utc": "2026-08-01T12:53:36.527880Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T125336519790Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T125336519790Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d44f83914c7925088be03f24263114865af8d43a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T125336519790Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8662895336ffbf16f7c0ebbc1c8bcb4cd308188c4dd460994c66b508c6afd46c +size 203826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T125337474041Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T125337474041Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2327dc3434b29f09522d49ac7caa15e36ea16394 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T125337474041Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133963638784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_peak": 17777560064, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149275279360, + "reserved_bytes_peak": 149275279360, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:37.473601Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115809880064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_peak": 17777560064, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149275279360, + "reserved_bytes_peak": 149275279360, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:36.528954Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17777560064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149275279360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149275279360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T125337474041Z.pickle", + "timestamp_utc": "2026-08-01T12:53:37.479963Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T125337474041Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T125337474041Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3cb54bd5d2b0598808447135cdce3a16df4fe332 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T125337474041Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:213af26a96473dc5c6b0742624d6b95dcf6dfd678a2550496fe3b91711635969 +size 203826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T125338426815Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T125338426815Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31938f3bf36d0a703b02f0f52489b7e806d4a548 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T125338426815Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133963638784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_peak": 17777560064, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149275279360, + "reserved_bytes_peak": 149275279360, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:38.426410Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115809880064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_peak": 17777560064, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149275279360, + "reserved_bytes_peak": 149275279360, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:37.480939Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17777560064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149275279360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149275279360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T125338426815Z.pickle", + "timestamp_utc": "2026-08-01T12:53:38.432562Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T125338426815Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T125338426815Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3dee0add0ec01be99cda4492314f3f0df4b102ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T125338426815Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:651c8c2ce9f22665427fe8beb7cb43329e449ea163caa40e0847f005e12ff5be +size 203826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T125437081172Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T125437081172Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b15b12fdbea8cbd27723a7a47a2c5bff8f15221e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T125437081172Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66956006400, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 133963638784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66956006400, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66956006400, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 5018250240, + "inactive_split_bytes_peak": 17777560064, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 149344485376, + "reserved_bytes_peak": 149344485376, + "segment_count_current": 684, + "segment_count_peak": 684, + "timestamp_utc": "2026-08-01T12:54:37.080635Z", + "world_size": 8 + }, + "allocated_bytes_current": 66956006400, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 115809880064, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133963638784, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115809880064, + "device": 3, + "inactive_split_bytes_current": 12335363072, + "inactive_split_bytes_peak": 17777560064, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 149275279360, + "reserved_bytes_peak": 149275279360, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:38.433481Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5018250240, + "inactive_split_bytes_current_delta": -7317112832, + "inactive_split_bytes_peak": 17777560064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 149344485376, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 149344485376, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 684, + "segment_count_current_delta": 5, + "segment_count_peak": 684, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T125437081172Z.pickle", + "timestamp_utc": "2026-08-01T12:54:37.167941Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T125437081172Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T125437081172Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f37818c0ae8eef5ff700f3289430c5eed98cdc6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T125437081172Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a4a21959d9fb1dd7ae022135be70e8e0d5fdb815defae9dcb7cd3ede8c6e32 +size 3862769 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T125328247452Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T125328247452Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f9563e7f6bb6ad73c3643af86aef674113c6f9c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T125328247452Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922719232, + "active_bytes_current_delta": 31690821632, + "active_bytes_peak": 92162351104, + "active_bytes_peak_delta": 56695572480, + "after": { + "active_bytes_current": 66922719232, + "active_bytes_peak": 92162351104, + "allocated_bytes_current": 66922719232, + "allocated_bytes_peak": 92162351104, + "device": 4, + "inactive_split_bytes_current": 7129815040, + "inactive_split_bytes_peak": 11155290112, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 98283028480, + "reserved_bytes_peak": 98283028480, + "segment_count_current": 450, + "segment_count_peak": 450, + "timestamp_utc": "2026-08-01T12:53:28.246946Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922719232, + "allocated_bytes_current_delta": 31690821632, + "allocated_bytes_peak": 92162351104, + "allocated_bytes_peak_delta": 56695572480, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:52:18.302185Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7129815040, + "inactive_split_bytes_current_delta": 6997438464, + "inactive_split_bytes_peak": 11155290112, + "inactive_split_bytes_peak_delta": 11001880576, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 98283028480, + "reserved_bytes_current_delta": 59039023104, + "reserved_bytes_peak": 98283028480, + "reserved_bytes_peak_delta": 59039023104, + "segment_count_current": 450, + "segment_count_current_delta": 379, + "segment_count_peak": 450, + "segment_count_peak_delta": 379, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T125328247452Z.pickle", + "timestamp_utc": "2026-08-01T12:53:28.361253Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T125328247452Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T125328247452Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e586a335bb2a3c391c505298b8c8701f189b06d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T125328247452Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3db4ba4ff70d75712bc67f21d4178a4a43aca2dec0bb32ec5f3e6ff85b93920a +size 5396174 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T125329660957Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T125329660957Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7a79cd0958b32c24a7bf6982b7d18b12b5e56b1c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T125329660957Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922719232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115809212416, + "active_bytes_peak_delta": 23646861312, + "after": { + "active_bytes_current": 66922719232, + "active_bytes_peak": 115809212416, + "allocated_bytes_current": 66922719232, + "allocated_bytes_peak": 115809212416, + "device": 4, + "inactive_split_bytes_current": 8847382528, + "inactive_split_bytes_peak": 11350207488, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117440512000, + "reserved_bytes_peak": 117440512000, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:29.660554Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922719232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115809212416, + "allocated_bytes_peak_delta": 23646861312, + "before": { + "active_bytes_current": 66922719232, + "active_bytes_peak": 92162351104, + "allocated_bytes_current": 66922719232, + "allocated_bytes_peak": 92162351104, + "device": 4, + "inactive_split_bytes_current": 7129815040, + "inactive_split_bytes_peak": 11155290112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 98283028480, + "reserved_bytes_peak": 98283028480, + "segment_count_current": 450, + "segment_count_peak": 450, + "timestamp_utc": "2026-08-01T12:53:28.378334Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8847382528, + "inactive_split_bytes_current_delta": 1717567488, + "inactive_split_bytes_peak": 11350207488, + "inactive_split_bytes_peak_delta": 194917376, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117440512000, + "reserved_bytes_current_delta": 19157483520, + "reserved_bytes_peak": 117440512000, + "reserved_bytes_peak_delta": 19157483520, + "segment_count_current": 499, + "segment_count_current_delta": 49, + "segment_count_peak": 499, + "segment_count_peak_delta": 49, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T125329660957Z.pickle", + "timestamp_utc": "2026-08-01T12:53:29.773003Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T125329660957Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T125329660957Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c33147cc38176011ff0723cb0cc51b837cfd077 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T125329660957Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a53bff314382493ba22be5e5ecdc3101831a0cd2a6a7c37058df1f2349a8cffa +size 5558319 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T125330823648Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T125330823648Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4aa3fc3c84e1f6f9ac764ba0467802ac3a393c86 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T125330823648Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922719232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115809212416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66922719232, + "active_bytes_peak": 115809212416, + "allocated_bytes_current": 66922719232, + "allocated_bytes_peak": 115809212416, + "device": 4, + "inactive_split_bytes_current": 8492963840, + "inactive_split_bytes_peak": 11350207488, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117440512000, + "reserved_bytes_peak": 117440512000, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:30.823248Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922719232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115809212416, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66922719232, + "active_bytes_peak": 115809212416, + "allocated_bytes_current": 66922719232, + "allocated_bytes_peak": 115809212416, + "device": 4, + "inactive_split_bytes_current": 8847382528, + "inactive_split_bytes_peak": 11350207488, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117440512000, + "reserved_bytes_peak": 117440512000, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:29.789065Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8492963840, + "inactive_split_bytes_current_delta": -354418688, + "inactive_split_bytes_peak": 11350207488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117440512000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117440512000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T125330823648Z.pickle", + "timestamp_utc": "2026-08-01T12:53:30.924261Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T125330823648Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T125330823648Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ee5e7dcff33c6d2533f07d12a5e1aa324af0054 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T125330823648Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e97da987e4cf94e8221fab011361a04bb534dd34c1088b3b6661174b7d4562dc +size 5576247 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T125333499439Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T125333499439Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c32d70d6a694217c4d57258d9a9d8e71f608a52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T125333499439Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66940680192, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 133971268608, + "active_bytes_peak_delta": 18162056192, + "after": { + "active_bytes_current": 66940680192, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66940680192, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_peak": 17613114368, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 149791178752, + "reserved_bytes_peak": 149791178752, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T12:53:33.498911Z", + "world_size": 8 + }, + "allocated_bytes_current": 66940680192, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 115817509888, + "allocated_bytes_peak_delta": 8297472, + "before": { + "active_bytes_current": 66922719232, + "active_bytes_peak": 115809212416, + "allocated_bytes_current": 66922719232, + "allocated_bytes_peak": 115809212416, + "device": 4, + "inactive_split_bytes_current": 8492963840, + "inactive_split_bytes_peak": 11350207488, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117440512000, + "reserved_bytes_peak": 117440512000, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:30.936958Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_current_delta": 5275250688, + "inactive_split_bytes_peak": 17613114368, + "inactive_split_bytes_peak_delta": 6262906880, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 149791178752, + "reserved_bytes_current_delta": 32350666752, + "reserved_bytes_peak": 149791178752, + "reserved_bytes_peak_delta": 32350666752, + "segment_count_current": 682, + "segment_count_current_delta": 183, + "segment_count_peak": 682, + "segment_count_peak_delta": 183, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T125333499439Z.pickle", + "timestamp_utc": "2026-08-01T12:53:33.617393Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T125333499439Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T125333499439Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c60d055723153d30e9f60592dde3df88194f9a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T125333499439Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b079c06548381dab6c53876d7f3e35234d728dc34fb3ba514a00dc5fd0aba1c +size 5822900 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T125334599541Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T125334599541Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aa9e0c120be75b43318300fbf382b5bcdaf6e921 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T125334599541Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66940680192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133971268608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66940680192, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66940680192, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_peak": 17613114368, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 149791178752, + "reserved_bytes_peak": 149791178752, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T12:53:34.599075Z", + "world_size": 8 + }, + "allocated_bytes_current": 66940680192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115817509888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66940680192, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66940680192, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_peak": 17613114368, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 149791178752, + "reserved_bytes_peak": 149791178752, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T12:53:33.631635Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17613114368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 149791178752, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149791178752, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 682, + "segment_count_current_delta": 0, + "segment_count_peak": 682, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T125334599541Z.pickle", + "timestamp_utc": "2026-08-01T12:53:34.605893Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T125334599541Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T125334599541Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b477390f1f9b383061be858e3143aa99e24546eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T125334599541Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8342ea5b07fe0db9200bf69a6aed1bc14dce711dff0a540ed85d40561931e11 +size 213430 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T125335555966Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T125335555966Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5db7a3a6d5e2de988cbb33dba4bbd4377e60e8fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T125335555966Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66940680192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133971268608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66940680192, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66940680192, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_peak": 17613114368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 149791178752, + "reserved_bytes_peak": 149791178752, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T12:53:35.555563Z", + "world_size": 8 + }, + "allocated_bytes_current": 66940680192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115817509888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66940680192, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66940680192, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_peak": 17613114368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 149791178752, + "reserved_bytes_peak": 149791178752, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T12:53:34.606850Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17613114368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 149791178752, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149791178752, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 682, + "segment_count_current_delta": 0, + "segment_count_peak": 682, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T125335555966Z.pickle", + "timestamp_utc": "2026-08-01T12:53:35.561906Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T125335555966Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T125335555966Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..78c52a3e326549101664bc561c53531b5bc1a1a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T125335555966Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34511d990a0844c3cfdbbe5818a73a06de131c15e5ebc6f24da4f8eb54fdd2c8 +size 213430 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T125336520943Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T125336520943Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c9d46afc40d8288ec893dc3852d773749be4e89 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T125336520943Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66940680192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133971268608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66940680192, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66940680192, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_peak": 17613114368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 149791178752, + "reserved_bytes_peak": 149791178752, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T12:53:36.519817Z", + "world_size": 8 + }, + "allocated_bytes_current": 66940680192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115817509888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66940680192, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66940680192, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_peak": 17613114368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 149791178752, + "reserved_bytes_peak": 149791178752, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T12:53:35.562805Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17613114368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 149791178752, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149791178752, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 682, + "segment_count_current_delta": 0, + "segment_count_peak": 682, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T125336520943Z.pickle", + "timestamp_utc": "2026-08-01T12:53:36.527324Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T125336520943Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T125336520943Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ca590b17b63b123c8f741523e42c14fdf655406 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T125336520943Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11f5e93863f1bd44c216e8e8e227db68ac536f687e26e74aa00d74fed35479a3 +size 213430 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T125337474183Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T125337474183Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e300032db9efe726587a1867131513725718a27e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T125337474183Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66940680192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133971268608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66940680192, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66940680192, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_peak": 17613114368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 149791178752, + "reserved_bytes_peak": 149791178752, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T12:53:37.473636Z", + "world_size": 8 + }, + "allocated_bytes_current": 66940680192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115817509888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66940680192, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66940680192, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_peak": 17613114368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 149791178752, + "reserved_bytes_peak": 149791178752, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T12:53:36.528247Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17613114368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 149791178752, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149791178752, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 682, + "segment_count_current_delta": 0, + "segment_count_peak": 682, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T125337474183Z.pickle", + "timestamp_utc": "2026-08-01T12:53:37.480466Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T125337474183Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T125337474183Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a0fb745a5d00931826fda6e5a14c5a80242a56a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T125337474183Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0281088e7a9cc6ae05e3c0e4540597f7302f6cf260e08dd345ccb8a56bd27034 +size 213430 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T125338426768Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T125338426768Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d108baa8a802deaa0feaf226b78f018b5a7d91e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T125338426768Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66940680192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133971268608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66940680192, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66940680192, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_peak": 17613114368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 149791178752, + "reserved_bytes_peak": 149791178752, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T12:53:38.426371Z", + "world_size": 8 + }, + "allocated_bytes_current": 66940680192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115817509888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66940680192, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66940680192, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_peak": 17613114368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 149791178752, + "reserved_bytes_peak": 149791178752, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T12:53:37.481429Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 17613114368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 149791178752, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149791178752, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 682, + "segment_count_current_delta": 0, + "segment_count_peak": 682, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T125338426768Z.pickle", + "timestamp_utc": "2026-08-01T12:53:38.432718Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T125338426768Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T125338426768Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e62bce92bef1de273dbb55e8b242fd8f1317f41 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T125338426768Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faec0b8d14c3da3aa74c499d4d7595f123d04a5689163904af7b0eadf029c5d8 +size 213430 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T125437081104Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T125437081104Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1537e8346fb13cb7142a4e43af0c29ba7e7cd2f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T125437081104Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66957606912, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 133971268608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66957606912, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66957606912, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 5335416832, + "inactive_split_bytes_peak": 17613114368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 149860384768, + "reserved_bytes_peak": 149860384768, + "segment_count_current": 687, + "segment_count_peak": 687, + "timestamp_utc": "2026-08-01T12:54:37.080631Z", + "world_size": 8 + }, + "allocated_bytes_current": 66957606912, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 115817509888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66940680192, + "active_bytes_peak": 133971268608, + "allocated_bytes_current": 66940680192, + "allocated_bytes_peak": 115817509888, + "device": 4, + "inactive_split_bytes_current": 13768214528, + "inactive_split_bytes_peak": 17613114368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 149791178752, + "reserved_bytes_peak": 149791178752, + "segment_count_current": 682, + "segment_count_peak": 682, + "timestamp_utc": "2026-08-01T12:53:38.433625Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5335416832, + "inactive_split_bytes_current_delta": -8432797696, + "inactive_split_bytes_peak": 17613114368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 149860384768, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 149860384768, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 687, + "segment_count_current_delta": 5, + "segment_count_peak": 687, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T125437081104Z.pickle", + "timestamp_utc": "2026-08-01T12:54:37.161766Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T125437081104Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T125437081104Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..33815d4f7c8361ed98f77a0f68a4cc5d3e8d9511 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T125437081104Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e49f7d1c29c16179da2c9f740a457226039d6c4d03a0989dad921bf5c92e6cf1 +size 4010809 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T125328247361Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T125328247361Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f6ac2e806eb7411cb99353323c6dbe9b3b077e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T125328247361Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 92162809856, + "active_bytes_peak_delta": 56696473600, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92162809856, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92162809856, + "device": 5, + "inactive_split_bytes_current": 7313867776, + "inactive_split_bytes_peak": 11256206336, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 98750693376, + "reserved_bytes_peak": 98750693376, + "segment_count_current": 453, + "segment_count_peak": 453, + "timestamp_utc": "2026-08-01T12:53:28.246884Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 92162809856, + "allocated_bytes_peak_delta": 56696473600, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:52:18.302405Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7313867776, + "inactive_split_bytes_current_delta": 7181048832, + "inactive_split_bytes_peak": 11256206336, + "inactive_split_bytes_peak_delta": 11095820288, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 98750693376, + "reserved_bytes_current_delta": 59506688000, + "reserved_bytes_peak": 98750693376, + "reserved_bytes_peak_delta": 59506688000, + "segment_count_current": 453, + "segment_count_current_delta": 382, + "segment_count_peak": 453, + "segment_count_peak_delta": 382, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T125328247361Z.pickle", + "timestamp_utc": "2026-08-01T12:53:28.357392Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T125328247361Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T125328247361Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..77991d6194016fba9dfde53cb4f94db376760f41 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T125328247361Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:891ad21eed57c98e5a1c591bdff4bdf85e52a6917b7fd2366a98093df1e0a8f3 +size 5534095 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T125329660919Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T125329660919Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d8d598ddebd2aee3085e3904e22a0885021bd67e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T125329660919Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115799111680, + "active_bytes_peak_delta": 23636301824, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115799111680, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115799111680, + "device": 5, + "inactive_split_bytes_current": 8786068480, + "inactive_split_bytes_peak": 11256206336, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:29.660512Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115799111680, + "allocated_bytes_peak_delta": 23636301824, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92162809856, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92162809856, + "device": 5, + "inactive_split_bytes_current": 7313867776, + "inactive_split_bytes_peak": 11256206336, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 98750693376, + "reserved_bytes_peak": 98750693376, + "segment_count_current": 453, + "segment_count_peak": 453, + "timestamp_utc": "2026-08-01T12:53:28.375208Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8786068480, + "inactive_split_bytes_current_delta": 1472200704, + "inactive_split_bytes_peak": 11256206336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118086434816, + "reserved_bytes_current_delta": 19335741440, + "reserved_bytes_peak": 118086434816, + "reserved_bytes_peak_delta": 19335741440, + "segment_count_current": 499, + "segment_count_current_delta": 46, + "segment_count_peak": 499, + "segment_count_peak_delta": 46, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T125329660919Z.pickle", + "timestamp_utc": "2026-08-01T12:53:29.765818Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T125329660919Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T125329660919Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4f16597ded762ae88e6c3efa47069a232a69b4ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T125329660919Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b66273656a6ee5e69a6927eb147546f23b65a516229f17ffc351f18a629495b8 +size 5532151 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T125330823645Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T125330823645Z.json new file mode 100644 index 0000000000000000000000000000000000000000..79fc414f331b773d9e9698d231352ec32a1a11c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T125330823645Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115803828224, + "active_bytes_peak_delta": 4716544, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115803828224, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115803828224, + "device": 5, + "inactive_split_bytes_current": 8494564352, + "inactive_split_bytes_peak": 11256206336, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:30.823240Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115803828224, + "allocated_bytes_peak_delta": 4716544, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115799111680, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115799111680, + "device": 5, + "inactive_split_bytes_current": 8786068480, + "inactive_split_bytes_peak": 11256206336, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:29.780517Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8494564352, + "inactive_split_bytes_current_delta": -291504128, + "inactive_split_bytes_peak": 11256206336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118086434816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118086434816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T125330823645Z.pickle", + "timestamp_utc": "2026-08-01T12:53:30.924125Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T125330823645Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T125330823645Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..784a076189a9fb125b251456c19c2e16f15ce354 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T125330823645Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9f286784e5d1be3d54040676ff4da76b16ad912d94aecc01e5db7d4735877a9 +size 5567412 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T125333499437Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T125333499437Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7601711fc590ea299a601d746ec554dd407c3666 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T125333499437Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 133958133760, + "active_bytes_peak_delta": 18154305536, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_peak": 16045609984, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149849899008, + "reserved_bytes_peak": 149849899008, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:53:33.498890Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 115804375040, + "allocated_bytes_peak_delta": 546816, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115803828224, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115803828224, + "device": 5, + "inactive_split_bytes_current": 8494564352, + "inactive_split_bytes_peak": 11256206336, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:30.936958Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_current_delta": 4373475328, + "inactive_split_bytes_peak": 16045609984, + "inactive_split_bytes_peak_delta": 4789403648, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149849899008, + "reserved_bytes_current_delta": 31763464192, + "reserved_bytes_peak": 149849899008, + "reserved_bytes_peak_delta": 31763464192, + "segment_count_current": 685, + "segment_count_current_delta": 186, + "segment_count_peak": 685, + "segment_count_peak_delta": 186, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T125333499437Z.pickle", + "timestamp_utc": "2026-08-01T12:53:33.616920Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T125333499437Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T125333499437Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..99dd8ff04bc27b45d3fbadcc5b2f2e455ff2b9ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T125333499437Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f98d32fd221e72dd626eb6d0bf972138861014fe08ff508cf327bd018d433f4f +size 5790482 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T125334599526Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T125334599526Z.json new file mode 100644 index 0000000000000000000000000000000000000000..edd11f4693a7add85eb338e98c3417664159810d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T125334599526Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133958133760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_peak": 16045609984, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149849899008, + "reserved_bytes_peak": 149849899008, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:53:34.599059Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115804375040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_peak": 16045609984, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149849899008, + "reserved_bytes_peak": 149849899008, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:53:33.633820Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16045609984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149849899008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149849899008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 685, + "segment_count_current_delta": 0, + "segment_count_peak": 685, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T125334599526Z.pickle", + "timestamp_utc": "2026-08-01T12:53:34.605634Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T125334599526Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T125334599526Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d6feae9d26a3cc5c6f93bb0677bebdb2b06eb17 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T125334599526Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff7a3100d11e030c02347a2f542b75be6b7721e7fe404688a25adffc0f043ec5 +size 203819 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T125335555944Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T125335555944Z.json new file mode 100644 index 0000000000000000000000000000000000000000..719f1ff0b9b0b5b46d007cd3d3588c9b57be750b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T125335555944Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133958133760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_peak": 16045609984, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149849899008, + "reserved_bytes_peak": 149849899008, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:53:35.555550Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115804375040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_peak": 16045609984, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149849899008, + "reserved_bytes_peak": 149849899008, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:53:34.606584Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16045609984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149849899008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149849899008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 685, + "segment_count_current_delta": 0, + "segment_count_peak": 685, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T125335555944Z.pickle", + "timestamp_utc": "2026-08-01T12:53:35.561575Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T125335555944Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T125335555944Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..62c03e72e7a3398955e9158967401790e0ee9239 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T125335555944Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c3c0fafb15a39c09358c459a305eb308ffe7641d102e88b09e7a76c39839d9d +size 203850 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T125336519644Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T125336519644Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67ea1c583c2086d019557f6e03fff209edc0cb4b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T125336519644Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133958133760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_peak": 16045609984, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149849899008, + "reserved_bytes_peak": 149849899008, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:53:36.519157Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115804375040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_peak": 16045609984, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149849899008, + "reserved_bytes_peak": 149849899008, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:53:35.562470Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16045609984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149849899008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149849899008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 685, + "segment_count_current_delta": 0, + "segment_count_peak": 685, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T125336519644Z.pickle", + "timestamp_utc": "2026-08-01T12:53:36.526268Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T125336519644Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T125336519644Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..853f4496399fe08029cbaf8aed185b78c19cfa3d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T125336519644Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:021962182128173b8ce512f37961e8b2cf3c82513e95005d21b1ba36b4f0ffce +size 203819 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T125337473935Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T125337473935Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ecf96bd5d63633b08f5e7965cb3363b7e6e2d422 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T125337473935Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133958133760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_peak": 16045609984, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149849899008, + "reserved_bytes_peak": 149849899008, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:53:37.473528Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115804375040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_peak": 16045609984, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149849899008, + "reserved_bytes_peak": 149849899008, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:53:36.527196Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16045609984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149849899008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149849899008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 685, + "segment_count_current_delta": 0, + "segment_count_peak": 685, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T125337473935Z.pickle", + "timestamp_utc": "2026-08-01T12:53:37.479673Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T125337473935Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T125337473935Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bdb621d083e205e147d6a2184f41fc13e08545ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T125337473935Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e257d8c2eef77a0043ba9cff2fc71e591247bfd6fb9f6b972efb409a1db38ca +size 203850 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T125338426737Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T125338426737Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8da1618f0df4544dfd9bb3d12688d054bfc35afe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T125338426737Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133958133760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_peak": 16045609984, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 149849899008, + "reserved_bytes_peak": 149849899008, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:53:38.426336Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115804375040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_peak": 16045609984, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149849899008, + "reserved_bytes_peak": 149849899008, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:53:37.480592Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16045609984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 149849899008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149849899008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 685, + "segment_count_current_delta": 0, + "segment_count_peak": 685, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T125338426737Z.pickle", + "timestamp_utc": "2026-08-01T12:53:38.432302Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T125338426737Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T125338426737Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf8e53d589a9ae8f754cdc39ec3d8ee4ec81c8bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T125338426737Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5216d50344f44e826114750415a2421cd70aa76918b0c802d51581d16ef12576 +size 203819 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T125437081117Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T125437081117Z.json new file mode 100644 index 0000000000000000000000000000000000000000..54327957c36989139b0fca6cd7bd2c806c2c5d6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T125437081117Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66956006400, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 133958133760, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66956006400, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66956006400, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 5083261952, + "inactive_split_bytes_peak": 16045609984, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150743285760, + "reserved_bytes_peak": 150743285760, + "segment_count_current": 691, + "segment_count_peak": 691, + "timestamp_utc": "2026-08-01T12:54:37.080641Z", + "world_size": 8 + }, + "allocated_bytes_current": 66956006400, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 115804375040, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133958133760, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115804375040, + "device": 5, + "inactive_split_bytes_current": 12868039680, + "inactive_split_bytes_peak": 16045609984, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 149849899008, + "reserved_bytes_peak": 149849899008, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:53:38.433204Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5083261952, + "inactive_split_bytes_current_delta": -7784777728, + "inactive_split_bytes_peak": 16045609984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150743285760, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150743285760, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 691, + "segment_count_current_delta": 6, + "segment_count_peak": 691, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T125437081117Z.pickle", + "timestamp_utc": "2026-08-01T12:54:37.169319Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T125437081117Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T125437081117Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..196364149e0efa1ebb8ee94bef5d1ad4be1aeeab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T125437081117Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c83fed5f0cd37425aa798723f0dee8a34be02ff12d99f67428bc7a750489c34c +size 4003435 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T125328247323Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T125328247323Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ff3b78bf77a29a82dc07017a317e43b6216fa59 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T125328247323Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 92162809856, + "active_bytes_peak_delta": 56696473600, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92162809856, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92162809856, + "device": 6, + "inactive_split_bytes_current": 7313867776, + "inactive_split_bytes_peak": 10666906624, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98750693376, + "reserved_bytes_peak": 98750693376, + "segment_count_current": 453, + "segment_count_peak": 453, + "timestamp_utc": "2026-08-01T12:53:28.246851Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 92162809856, + "allocated_bytes_peak_delta": 56696473600, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:52:18.302924Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7313867776, + "inactive_split_bytes_current_delta": 7181048832, + "inactive_split_bytes_peak": 10666906624, + "inactive_split_bytes_peak_delta": 10506520576, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98750693376, + "reserved_bytes_current_delta": 59506688000, + "reserved_bytes_peak": 98750693376, + "reserved_bytes_peak_delta": 59506688000, + "segment_count_current": 453, + "segment_count_current_delta": 382, + "segment_count_peak": 453, + "segment_count_peak_delta": 382, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T125328247323Z.pickle", + "timestamp_utc": "2026-08-01T12:53:28.357431Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T125328247323Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T125328247323Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..59946c6e408c488479c700d59c54c96c899744dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T125328247323Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:946fc9eda123861efe05cd688204212bd8dc6f1c5ec24b8895ff040ddfb3ca6d +size 5534099 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T125329660893Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T125329660893Z.json new file mode 100644 index 0000000000000000000000000000000000000000..391ac50c3a210cacc00b865fc7bf8a99f6ef9d1b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T125329660893Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115799111680, + "active_bytes_peak_delta": 23636301824, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115799111680, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115799111680, + "device": 6, + "inactive_split_bytes_current": 8962229248, + "inactive_split_bytes_peak": 11053521920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:29.660482Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115799111680, + "allocated_bytes_peak_delta": 23636301824, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92162809856, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92162809856, + "device": 6, + "inactive_split_bytes_current": 7313867776, + "inactive_split_bytes_peak": 10666906624, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98750693376, + "reserved_bytes_peak": 98750693376, + "segment_count_current": 453, + "segment_count_peak": 453, + "timestamp_utc": "2026-08-01T12:53:28.375021Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8962229248, + "inactive_split_bytes_current_delta": 1648361472, + "inactive_split_bytes_peak": 11053521920, + "inactive_split_bytes_peak_delta": 386615296, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118086434816, + "reserved_bytes_current_delta": 19335741440, + "reserved_bytes_peak": 118086434816, + "reserved_bytes_peak_delta": 19335741440, + "segment_count_current": 499, + "segment_count_current_delta": 46, + "segment_count_peak": 499, + "segment_count_peak_delta": 46, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T125329660893Z.pickle", + "timestamp_utc": "2026-08-01T12:53:29.766626Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T125329660893Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T125329660893Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e1a1efdeed51be3e0f81c75d5e01142ebec4367 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T125329660893Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67038fc1c216f7cff7283303314b00c02c51dd6bfa97e4f72116eed1ef8f749c +size 5535434 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T125330823638Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T125330823638Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9adc91892aeefc2328e271a7a45188c90c9f43f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T125330823638Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115806430208, + "active_bytes_peak_delta": 7318528, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115806430208, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 7490028544, + "inactive_split_bytes_peak": 11053521920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:30.823225Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806430208, + "allocated_bytes_peak_delta": 7318528, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115799111680, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115799111680, + "device": 6, + "inactive_split_bytes_current": 8962229248, + "inactive_split_bytes_peak": 11053521920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:29.781303Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7490028544, + "inactive_split_bytes_current_delta": -1472200704, + "inactive_split_bytes_peak": 11053521920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118086434816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118086434816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T125330823638Z.pickle", + "timestamp_utc": "2026-08-01T12:53:30.925727Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T125330823638Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T125330823638Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13713c20585cf39d80d23e0a500ad6eec12744c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T125330823638Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51b6474893aaae3c1412905047882ac867798ba69f5c7e287c92c3710b860c61 +size 5546947 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T125333499446Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T125333499446Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1b75f10f9cb1b111a11915f4bcb5023c757a01 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T125333499446Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 133959968768, + "active_bytes_peak_delta": 18153538560, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_peak": 16606303232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 148610482176, + "reserved_bytes_peak": 148610482176, + "segment_count_current": 677, + "segment_count_peak": 677, + "timestamp_utc": "2026-08-01T12:53:33.498900Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 115806430208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115806430208, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 7490028544, + "inactive_split_bytes_peak": 11053521920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:30.938516Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_current_delta": 4675465216, + "inactive_split_bytes_peak": 16606303232, + "inactive_split_bytes_peak_delta": 5552781312, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 148610482176, + "reserved_bytes_current_delta": 30524047360, + "reserved_bytes_peak": 148610482176, + "reserved_bytes_peak_delta": 30524047360, + "segment_count_current": 677, + "segment_count_current_delta": 178, + "segment_count_peak": 677, + "segment_count_peak_delta": 178, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T125333499446Z.pickle", + "timestamp_utc": "2026-08-01T12:53:33.615384Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T125333499446Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T125333499446Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d867ae45c7137ec186cf8fcdf6eab9a30f3a93ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T125333499446Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:518dfdb5aa48151535d64e342fd8a2eb7e45e9dc70b8cdceba9775d76f3c48dd +size 5827046 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T125334599526Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T125334599526Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab888dd7b4bfc1439c09eafdc28bafb3b01f3a0a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T125334599526Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133959968768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_peak": 16606303232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 148610482176, + "reserved_bytes_peak": 148610482176, + "segment_count_current": 677, + "segment_count_peak": 677, + "timestamp_utc": "2026-08-01T12:53:34.599056Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806430208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_peak": 16606303232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 148610482176, + "reserved_bytes_peak": 148610482176, + "segment_count_current": 677, + "segment_count_peak": 677, + "timestamp_utc": "2026-08-01T12:53:33.629835Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16606303232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 148610482176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148610482176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 677, + "segment_count_current_delta": 0, + "segment_count_peak": 677, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T125334599526Z.pickle", + "timestamp_utc": "2026-08-01T12:53:34.605666Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T125334599526Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T125334599526Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8c5589f222b40c8b3f777c0cba2330dc3dc16610 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T125334599526Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e809afcd42f5c7bbfd753b43a7fde0807ccbc0886f6d1325ea77f8bb27262da8 +size 204936 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T125335555962Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T125335555962Z.json new file mode 100644 index 0000000000000000000000000000000000000000..716fa01d0d81c446e2b4fab1ddb50cfe71cfa80b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T125335555962Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133959968768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_peak": 16606303232, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 148610482176, + "reserved_bytes_peak": 148610482176, + "segment_count_current": 677, + "segment_count_peak": 677, + "timestamp_utc": "2026-08-01T12:53:35.555544Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806430208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_peak": 16606303232, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 148610482176, + "reserved_bytes_peak": 148610482176, + "segment_count_current": 677, + "segment_count_peak": 677, + "timestamp_utc": "2026-08-01T12:53:34.606672Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16606303232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 148610482176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148610482176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 677, + "segment_count_current_delta": 0, + "segment_count_peak": 677, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T125335555962Z.pickle", + "timestamp_utc": "2026-08-01T12:53:35.561847Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T125335555962Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T125335555962Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..92d5e47b9d816d29cb0ba7426c4c432649930637 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T125335555962Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53e29c55a69a749b6048dd9b33d29136219dc47ef22a85f52a0030ef7c7d0945 +size 204936 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T125336519611Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T125336519611Z.json new file mode 100644 index 0000000000000000000000000000000000000000..abe4e3f6a090d6b78f0cf9a29cc113edb62e241f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T125336519611Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133959968768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_peak": 16606303232, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 148610482176, + "reserved_bytes_peak": 148610482176, + "segment_count_current": 677, + "segment_count_peak": 677, + "timestamp_utc": "2026-08-01T12:53:36.519111Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806430208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_peak": 16606303232, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 148610482176, + "reserved_bytes_peak": 148610482176, + "segment_count_current": 677, + "segment_count_peak": 677, + "timestamp_utc": "2026-08-01T12:53:35.562759Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16606303232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 148610482176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148610482176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 677, + "segment_count_current_delta": 0, + "segment_count_peak": 677, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T125336519611Z.pickle", + "timestamp_utc": "2026-08-01T12:53:36.526192Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T125336519611Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T125336519611Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..30447beef4395d59df95c441dbd170756d4b246e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T125336519611Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3c72c6e37358d08453f51c53c118e6d92025ebcd9549b5df3de0cc0928f7f6e +size 204936 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T125337473920Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T125337473920Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5bdf9ea6896ba255c1426fbda63646100f95a2b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T125337473920Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133959968768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_peak": 16606303232, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 148610482176, + "reserved_bytes_peak": 148610482176, + "segment_count_current": 677, + "segment_count_peak": 677, + "timestamp_utc": "2026-08-01T12:53:37.473508Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806430208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_peak": 16606303232, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 148610482176, + "reserved_bytes_peak": 148610482176, + "segment_count_current": 677, + "segment_count_peak": 677, + "timestamp_utc": "2026-08-01T12:53:36.527072Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16606303232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 148610482176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148610482176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 677, + "segment_count_current_delta": 0, + "segment_count_peak": 677, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T125337473920Z.pickle", + "timestamp_utc": "2026-08-01T12:53:37.479525Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T125337473920Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T125337473920Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29794f02008e7ece8b5535c97b098d0d4a15f684 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T125337473920Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:690caf63bee82ed452f88de75da6cb46579932389baa5edef1fe92d3170f745b +size 204936 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T125338426762Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T125338426762Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc37381f2c9541442f44d2377b11f83f6c3414d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T125338426762Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939079680, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133959968768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_peak": 16606303232, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 148610482176, + "reserved_bytes_peak": 148610482176, + "segment_count_current": 677, + "segment_count_peak": 677, + "timestamp_utc": "2026-08-01T12:53:38.426359Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939079680, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806430208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_peak": 16606303232, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 148610482176, + "reserved_bytes_peak": 148610482176, + "segment_count_current": 677, + "segment_count_peak": 677, + "timestamp_utc": "2026-08-01T12:53:37.480473Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16606303232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 148610482176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148610482176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 677, + "segment_count_current_delta": 0, + "segment_count_peak": 677, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T125338426762Z.pickle", + "timestamp_utc": "2026-08-01T12:53:38.432409Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T125338426762Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T125338426762Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..294aeef9ad550899e1f9d1e52e10b9f78c2e6733 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T125338426762Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:234697e809995e3b9218363513bb135bc6db91c4da4bd479b824d9f059da9fdb +size 204936 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T125437081110Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T125437081110Z.json new file mode 100644 index 0000000000000000000000000000000000000000..10d9b2472a43bc7023e92973d4167422a1054987 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T125437081110Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66956006400, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 133959968768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66956006400, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66956006400, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 4256984064, + "inactive_split_bytes_peak": 16606303232, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 149503868928, + "reserved_bytes_peak": 149503868928, + "segment_count_current": 683, + "segment_count_peak": 683, + "timestamp_utc": "2026-08-01T12:54:37.080643Z", + "world_size": 8 + }, + "allocated_bytes_current": 66956006400, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 115806430208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939079680, + "active_bytes_peak": 133959968768, + "allocated_bytes_current": 66939079680, + "allocated_bytes_peak": 115806430208, + "device": 6, + "inactive_split_bytes_current": 12165493760, + "inactive_split_bytes_peak": 16606303232, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 148610482176, + "reserved_bytes_peak": 148610482176, + "segment_count_current": 677, + "segment_count_peak": 677, + "timestamp_utc": "2026-08-01T12:53:38.433304Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4256984064, + "inactive_split_bytes_current_delta": -7908509696, + "inactive_split_bytes_peak": 16606303232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 149503868928, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 149503868928, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 683, + "segment_count_current_delta": 6, + "segment_count_peak": 683, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T125437081110Z.pickle", + "timestamp_utc": "2026-08-01T12:54:37.168537Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T125437081110Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T125437081110Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fea59bd4076ac919222bf26d627fa14e6fb367bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T125437081110Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e7a6378489aa223a59ed031b44fc7a026f0b1ec08a9acacf82a513337991eea +size 4002899 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T125328247467Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T125328247467Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4c0d18e80e533c31bb7c8dc51c5e79d7ead0e92b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T125328247467Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 31689664000, + "active_bytes_peak": 92161499136, + "active_bytes_peak_delta": 56695162880, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 92161499136, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 92161499136, + "device": 7, + "inactive_split_bytes_current": 7315964416, + "inactive_split_bytes_peak": 10845950976, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 98750693376, + "reserved_bytes_peak": 98750693376, + "segment_count_current": 453, + "segment_count_peak": 453, + "timestamp_utc": "2026-08-01T12:53:28.246975Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 31689664000, + "allocated_bytes_peak": 92161499136, + "allocated_bytes_peak_delta": 56695162880, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:52:18.304057Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7315964416, + "inactive_split_bytes_current_delta": 7183145472, + "inactive_split_bytes_peak": 10845950976, + "inactive_split_bytes_peak_delta": 10685564928, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 98750693376, + "reserved_bytes_current_delta": 59506688000, + "reserved_bytes_peak": 98750693376, + "reserved_bytes_peak_delta": 59506688000, + "segment_count_current": 453, + "segment_count_current_delta": 382, + "segment_count_peak": 453, + "segment_count_peak_delta": 382, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T125328247467Z.pickle", + "timestamp_utc": "2026-08-01T12:53:28.362818Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T125328247467Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T125328247467Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b01b580c8ed367f065e686c48fe4817e7335eaa0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T125328247467Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ace3c4acc65caa95b2265466f2627861cc34da5f063609baa1093f98447bd2de +size 5535365 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T125329661012Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T125329661012Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad9842d477af37cc0ed7d7bddbc3e31190cb7095 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T125329661012Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115808769536, + "active_bytes_peak_delta": 23647270400, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 115808769536, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 115808769536, + "device": 7, + "inactive_split_bytes_current": 9138389504, + "inactive_split_bytes_peak": 10845950976, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:29.660556Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115808769536, + "allocated_bytes_peak_delta": 23647270400, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 92161499136, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 92161499136, + "device": 7, + "inactive_split_bytes_current": 7315964416, + "inactive_split_bytes_peak": 10845950976, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 98750693376, + "reserved_bytes_peak": 98750693376, + "segment_count_current": 453, + "segment_count_peak": 453, + "timestamp_utc": "2026-08-01T12:53:28.381625Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9138389504, + "inactive_split_bytes_current_delta": 1822425088, + "inactive_split_bytes_peak": 10845950976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118086434816, + "reserved_bytes_current_delta": 19335741440, + "reserved_bytes_peak": 118086434816, + "reserved_bytes_peak_delta": 19335741440, + "segment_count_current": 499, + "segment_count_current_delta": 46, + "segment_count_peak": 499, + "segment_count_peak_delta": 46, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T125329661012Z.pickle", + "timestamp_utc": "2026-08-01T12:53:29.767180Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T125329661012Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T125329661012Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f686312040d504212f8fbe98c32e91b988b42d84 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T125329661012Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68dd663a83cad73c25cef637366a53bd5e146b806b4e8f3905fbd64d9ab232d0 +size 5536450 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T125330823685Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T125330823685Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0b7c70db24df6b3dc35cbe3f72f841ed23cdb976 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T125330823685Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115810746880, + "active_bytes_peak_delta": 1977344, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 115810746880, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 8194671104, + "inactive_split_bytes_peak": 10845950976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:30.823276Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115810746880, + "allocated_bytes_peak_delta": 1977344, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 115808769536, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 115808769536, + "device": 7, + "inactive_split_bytes_current": 9138389504, + "inactive_split_bytes_peak": 10845950976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:29.782396Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8194671104, + "inactive_split_bytes_current_delta": -943718400, + "inactive_split_bytes_peak": 10845950976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118086434816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118086434816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T125330823685Z.pickle", + "timestamp_utc": "2026-08-01T12:53:30.928055Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T125330823685Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T125330823685Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4320f8e3b792530000e13aa2156ef2ff5ebd0030 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T125330823685Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f22df3d9d8eaa1e74f16c0bd18226165e52c04a201d04601eeee30f7115ff5a7 +size 5553450 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T125333499452Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T125333499452Z.json new file mode 100644 index 0000000000000000000000000000000000000000..893a2a16c496a62aac4d8fd6ecd24fb4847d0df7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T125333499452Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939080192, + "active_bytes_current_delta": 17960960, + "active_bytes_peak": 133963115008, + "active_bytes_peak_delta": 18152368128, + "after": { + "active_bytes_current": 66939080192, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66939080192, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_peak": 16600610304, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149684224000, + "reserved_bytes_peak": 149684224000, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:33.498926Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939080192, + "allocated_bytes_current_delta": 17960960, + "allocated_bytes_peak": 115810746880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 115810746880, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 8194671104, + "inactive_split_bytes_peak": 10845950976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118086434816, + "reserved_bytes_peak": 118086434816, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T12:53:30.942694Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_current_delta": 4602064896, + "inactive_split_bytes_peak": 16600610304, + "inactive_split_bytes_peak_delta": 5754659328, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149684224000, + "reserved_bytes_current_delta": 31597789184, + "reserved_bytes_peak": 149684224000, + "reserved_bytes_peak_delta": 31597789184, + "segment_count_current": 679, + "segment_count_current_delta": 180, + "segment_count_peak": 679, + "segment_count_peak_delta": 180, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T125333499452Z.pickle", + "timestamp_utc": "2026-08-01T12:53:33.618084Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T125333499452Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T125333499452Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..01803dc054385049af22952289ee9798c27a8d19 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T125333499452Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c6225d63b4341f571ade2a63bdb7c89f147bfe190c9a01669af4359533aadf4 +size 5827609 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T125334599628Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T125334599628Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bebbbadc5cdbe118eb3b7ec4a781725c59b35d5c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T125334599628Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939080192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133963115008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939080192, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66939080192, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_peak": 16600610304, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149684224000, + "reserved_bytes_peak": 149684224000, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:34.599138Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939080192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115810746880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939080192, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66939080192, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_peak": 16600610304, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149684224000, + "reserved_bytes_peak": 149684224000, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:33.634477Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16600610304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149684224000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149684224000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T125334599628Z.pickle", + "timestamp_utc": "2026-08-01T12:53:34.606209Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T125334599628Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T125334599628Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..664ce4b3b94821fa929dbe3c635b8316cdb7302a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T125334599628Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1247f8ef93c464d170ada1a0f1ea9f114cdcced2bb87fe6ce9789d5ed8974364 +size 207560 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T125335556015Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T125335556015Z.json new file mode 100644 index 0000000000000000000000000000000000000000..25cfa749af470722439efb74a1a1283ce8ba9593 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T125335556015Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939080192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133963115008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939080192, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66939080192, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_peak": 16600610304, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149684224000, + "reserved_bytes_peak": 149684224000, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:35.555600Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939080192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115810746880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939080192, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66939080192, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_peak": 16600610304, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149684224000, + "reserved_bytes_peak": 149684224000, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:34.607218Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16600610304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149684224000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149684224000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T125335556015Z.pickle", + "timestamp_utc": "2026-08-01T12:53:35.562046Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T125335556015Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T125335556015Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a6c230fca47e224c391b25c3b6b73bd47c5e10fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T125335556015Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c3da457462c63a5f806e017c7a180cfcb4c642e8a46c9505794a0aecdef686d +size 207591 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T125336519629Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T125336519629Z.json new file mode 100644 index 0000000000000000000000000000000000000000..533aa5576f0fb146c17ee71aa52d8e6d3a83cd95 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T125336519629Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939080192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133963115008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939080192, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66939080192, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_peak": 16600610304, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149684224000, + "reserved_bytes_peak": 149684224000, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:36.519196Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939080192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115810746880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939080192, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66939080192, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_peak": 16600610304, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149684224000, + "reserved_bytes_peak": 149684224000, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:35.562957Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16600610304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149684224000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149684224000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T125336519629Z.pickle", + "timestamp_utc": "2026-08-01T12:53:36.525646Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T125336519629Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T125336519629Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72a9c117d17d2065f49add9d48667925be29df85 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T125336519629Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e499e5738834404c7c9edc4cce6a5882a42726b2cb2b671f7177b1b6bd36a096 +size 207591 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T125337474008Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T125337474008Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2140cc00c4410bfaa0fb3349053e270e5616469c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T125337474008Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939080192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133963115008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939080192, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66939080192, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_peak": 16600610304, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149684224000, + "reserved_bytes_peak": 149684224000, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:37.473595Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939080192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115810746880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939080192, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66939080192, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_peak": 16600610304, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149684224000, + "reserved_bytes_peak": 149684224000, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:36.526544Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16600610304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149684224000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149684224000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T125337474008Z.pickle", + "timestamp_utc": "2026-08-01T12:53:37.480181Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T125337474008Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T125337474008Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..216f13a728aa37de33417dda9ae898cc7137a2f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T125337474008Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d191c2fa1e3691fb8ce2f31fb445cdc5a8dac8621076c7ef5ee2a640bc6868a +size 207560 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T125338426867Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T125338426867Z.json new file mode 100644 index 0000000000000000000000000000000000000000..73285d8b6f994d165d3a14b96899c6599d66c85d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T125338426867Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939080192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133963115008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939080192, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66939080192, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_peak": 16600610304, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 149684224000, + "reserved_bytes_peak": 149684224000, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:38.426465Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939080192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115810746880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939080192, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66939080192, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_peak": 16600610304, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149684224000, + "reserved_bytes_peak": 149684224000, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:37.481116Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 16600610304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 149684224000, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 149684224000, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 679, + "segment_count_current_delta": 0, + "segment_count_peak": 679, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T125338426867Z.pickle", + "timestamp_utc": "2026-08-01T12:53:38.433222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T125338426867Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T125338426867Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d56504d57393712665ed2a7a76e18ee7bcb4bc2e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T125338426867Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d290f596086dea9579e9ceb8f5cc4edafad53c963ca7fe71414a4362cd6b828 +size 207591 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T125437081111Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T125437081111Z.json new file mode 100644 index 0000000000000000000000000000000000000000..25a4f4d6ea849470c0dbb6ce946b353f5b77d8f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T125437081111Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66956006912, + "active_bytes_current_delta": 16926720, + "active_bytes_peak": 133963115008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66956006912, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66956006912, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 4844186112, + "inactive_split_bytes_peak": 16600610304, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 150577610752, + "reserved_bytes_peak": 150577610752, + "segment_count_current": 685, + "segment_count_peak": 685, + "timestamp_utc": "2026-08-01T12:54:37.080642Z", + "world_size": 8 + }, + "allocated_bytes_current": 66956006912, + "allocated_bytes_current_delta": 16926720, + "allocated_bytes_peak": 115810746880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939080192, + "active_bytes_peak": 133963115008, + "allocated_bytes_current": 66939080192, + "allocated_bytes_peak": 115810746880, + "device": 7, + "inactive_split_bytes_current": 12796736000, + "inactive_split_bytes_peak": 16600610304, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 149684224000, + "reserved_bytes_peak": 149684224000, + "segment_count_current": 679, + "segment_count_peak": 679, + "timestamp_utc": "2026-08-01T12:53:38.434130Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4844186112, + "inactive_split_bytes_current_delta": -7952549888, + "inactive_split_bytes_peak": 16600610304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 150577610752, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 150577610752, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 685, + "segment_count_current_delta": 6, + "segment_count_peak": 685, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T125437081111Z.pickle", + "timestamp_utc": "2026-08-01T12:54:37.167497Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T125437081111Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T125437081111Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b80445d85e7ab9ee1f854193b80569cef0550e2a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T125437081111Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:900169d2716d9c07a091028ac517fb59e7db3549bfd8cab68152e6f51dffdf4a +size 3944288 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..df283a967e7d0816003917f50b0f30a37dbe3e02 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: on +seq_length: 4096 +capacity: 1.50 +expert_dispatch_overlap: off +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6021 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 241 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T125731533992Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T125731533992Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c46a5ee3648d4b62b60c7fdc93a9bed536638f15 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T125731533992Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509799424, + "active_bytes_current_delta": 37278344192, + "active_bytes_peak": 92053241344, + "active_bytes_peak_delta": 56586905088, + "after": { + "active_bytes_current": 72509799424, + "active_bytes_peak": 92053241344, + "allocated_bytes_current": 72509799424, + "allocated_bytes_peak": 92053241344, + "device": 0, + "inactive_split_bytes_current": 6496207872, + "inactive_split_bytes_peak": 12346124800, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 95638519808, + "reserved_bytes_peak": 95638519808, + "segment_count_current": 373, + "segment_count_peak": 373, + "timestamp_utc": "2026-08-01T12:57:31.533285Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509799424, + "allocated_bytes_current_delta": 37278344192, + "allocated_bytes_peak": 92053241344, + "allocated_bytes_peak_delta": 56586905088, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T12:56:23.360928Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6496207872, + "inactive_split_bytes_current_delta": 6361291776, + "inactive_split_bytes_peak": 12346124800, + "inactive_split_bytes_peak_delta": 12185492992, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 95638519808, + "reserved_bytes_current_delta": 56392417280, + "reserved_bytes_peak": 95638519808, + "reserved_bytes_peak_delta": 56392417280, + "segment_count_current": 373, + "segment_count_current_delta": 301, + "segment_count_peak": 373, + "segment_count_peak_delta": 301, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T125731533992Z.pickle", + "timestamp_utc": "2026-08-01T12:57:31.663902Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T125731533992Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T125731533992Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3612948d74a9f73d82ca42da3c8fb6eeba069e9a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T125731533992Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6f3104aa94a79e326ca4a019c49ca5e8207456e1b4d3df4a0f1af389dbccf01 +size 5921565 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T125732832984Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T125732832984Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9451adcec2e1a70b7f5d17ffa3f83e92504ad697 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T125732832984Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509799424, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115700348928, + "active_bytes_peak_delta": 23647107584, + "after": { + "active_bytes_current": 72509799424, + "active_bytes_peak": 115700348928, + "allocated_bytes_current": 72509799424, + "allocated_bytes_peak": 115700348928, + "device": 0, + "inactive_split_bytes_current": 7381206016, + "inactive_split_bytes_peak": 15657885696, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118694608896, + "reserved_bytes_peak": 118694608896, + "segment_count_current": 429, + "segment_count_peak": 429, + "timestamp_utc": "2026-08-01T12:57:32.832567Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509799424, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115700348928, + "allocated_bytes_peak_delta": 23647107584, + "before": { + "active_bytes_current": 72509799424, + "active_bytes_peak": 92053241344, + "allocated_bytes_current": 72509799424, + "allocated_bytes_peak": 92053241344, + "device": 0, + "inactive_split_bytes_current": 6496207872, + "inactive_split_bytes_peak": 12346124800, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 95638519808, + "reserved_bytes_peak": 95638519808, + "segment_count_current": 373, + "segment_count_peak": 373, + "timestamp_utc": "2026-08-01T12:57:31.685750Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7381206016, + "inactive_split_bytes_current_delta": 884998144, + "inactive_split_bytes_peak": 15657885696, + "inactive_split_bytes_peak_delta": 3311760896, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118694608896, + "reserved_bytes_current_delta": 23056089088, + "reserved_bytes_peak": 118694608896, + "reserved_bytes_peak_delta": 23056089088, + "segment_count_current": 429, + "segment_count_current_delta": 56, + "segment_count_peak": 429, + "segment_count_peak_delta": 56, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T125732832984Z.pickle", + "timestamp_utc": "2026-08-01T12:57:32.946540Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T125732832984Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T125732832984Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..176f7ef9472d3ceeb656dd522008dd7be9909a07 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T125732832984Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49bd74ef5ebb04f8dc6e3bd5d48feed68331af7d89079ab38d5c73b47fc25a0a +size 6578198 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T125734133051Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T125734133051Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f0c6df18f6b844a218bd68eadb8fcfd526a80d69 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T125734133051Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72510848000, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115709032448, + "active_bytes_peak_delta": 8683520, + "after": { + "active_bytes_current": 72510848000, + "active_bytes_peak": 115709032448, + "allocated_bytes_current": 72510848000, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 8202241024, + "inactive_split_bytes_peak": 15657885696, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118694608896, + "reserved_bytes_peak": 118694608896, + "segment_count_current": 429, + "segment_count_peak": 429, + "timestamp_utc": "2026-08-01T12:57:34.132636Z", + "world_size": 8 + }, + "allocated_bytes_current": 72510848000, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115709032448, + "allocated_bytes_peak_delta": 8683520, + "before": { + "active_bytes_current": 72509799424, + "active_bytes_peak": 115700348928, + "allocated_bytes_current": 72509799424, + "allocated_bytes_peak": 115700348928, + "device": 0, + "inactive_split_bytes_current": 7381206016, + "inactive_split_bytes_peak": 15657885696, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118694608896, + "reserved_bytes_peak": 118694608896, + "segment_count_current": 429, + "segment_count_peak": 429, + "timestamp_utc": "2026-08-01T12:57:32.962915Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8202241024, + "inactive_split_bytes_current_delta": 821035008, + "inactive_split_bytes_peak": 15657885696, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118694608896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118694608896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 429, + "segment_count_current_delta": 0, + "segment_count_peak": 429, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T125734133051Z.pickle", + "timestamp_utc": "2026-08-01T12:57:34.267599Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T125734133051Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T125734133051Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0f541475e9812a4b95ecabc5e7fa55ef7091f24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T125734133051Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8393edcfa0aeaac7f281bea3cd9c47940e2dc070ff3eefca1a5cf260d1752fd5 +size 6593020 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T125737023077Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T125737023077Z.json new file mode 100644 index 0000000000000000000000000000000000000000..00b0aa0634aff09316e88ccd2760a2bcf842d02b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T125737023077Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526842880, + "active_bytes_current_delta": 15994880, + "active_bytes_peak": 133861656576, + "active_bytes_peak_delta": 18152624128, + "after": { + "active_bytes_current": 72526842880, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 72526842880, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_peak": 19863188480, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_peak": 150695051264, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T12:57:37.022259Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526842880, + "allocated_bytes_current_delta": 15994880, + "allocated_bytes_peak": 115709032448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72510848000, + "active_bytes_peak": 115709032448, + "allocated_bytes_current": 72510848000, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 8202241024, + "inactive_split_bytes_peak": 15657885696, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118694608896, + "reserved_bytes_peak": 118694608896, + "segment_count_current": 429, + "segment_count_peak": 429, + "timestamp_utc": "2026-08-01T12:57:34.292837Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_current_delta": 11602227200, + "inactive_split_bytes_peak": 19863188480, + "inactive_split_bytes_peak_delta": 4205302784, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_current_delta": 32000442368, + "reserved_bytes_peak": 150695051264, + "reserved_bytes_peak_delta": 32000442368, + "segment_count_current": 550, + "segment_count_current_delta": 121, + "segment_count_peak": 550, + "segment_count_peak_delta": 121, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T125737023077Z.pickle", + "timestamp_utc": "2026-08-01T12:57:37.147336Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T125737023077Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T125737023077Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f57c6f67da9bce55734cead180ab20a0da2b6333 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T125737023077Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca20d434f8d27e3cc4c28a891c34c7fd26c8cb1158e6f298a1166cf53e1b089d +size 6327082 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T125738257745Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T125738257745Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4c9c4b4cc8ce9b79fa7175321e34233a4a52b70b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T125738257745Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526842880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133861656576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526842880, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 72526842880, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_peak": 19863188480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_peak": 150695051264, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T12:57:38.256729Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526842880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115709032448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526842880, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 72526842880, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_peak": 19863188480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_peak": 150695051264, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T12:57:37.163409Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19863188480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150695051264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 550, + "segment_count_current_delta": 0, + "segment_count_peak": 550, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T125738257745Z.pickle", + "timestamp_utc": "2026-08-01T12:57:38.262979Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T125738257745Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T125738257745Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4f9a836a5ccd99530ad48ac6eddd8d174f6befa6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T125738257745Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abfc30023bddc0e6498df59f71a62ba64d568e44bcbde63bb69a5c47f5a3a294 +size 147291 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T125739347460Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T125739347460Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f7e6cca8e6ffa2514cf407662d5d8d2ad4d5e54b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T125739347460Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526842880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133861656576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526842880, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 72526842880, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_peak": 19863188480, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_peak": 150695051264, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T12:57:39.346999Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526842880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115709032448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526842880, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 72526842880, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_peak": 19863188480, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_peak": 150695051264, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T12:57:38.264038Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19863188480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150695051264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 550, + "segment_count_current_delta": 0, + "segment_count_peak": 550, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T125739347460Z.pickle", + "timestamp_utc": "2026-08-01T12:57:39.351796Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T125739347460Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T125739347460Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f6fe848f20426d3594033cec64dfae3a09fdcca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T125739347460Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9970c580e0ff526c6779a00d9ae416d0dcc54d68bff79443a22a996d31132d3c +size 147291 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T125740440644Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T125740440644Z.json new file mode 100644 index 0000000000000000000000000000000000000000..73a75f373be7c8b3720b788569b645398cc0a712 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T125740440644Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526842880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133861656576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526842880, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 72526842880, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_peak": 19863188480, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_peak": 150695051264, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T12:57:40.440155Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526842880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115709032448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526842880, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 72526842880, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_peak": 19863188480, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_peak": 150695051264, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T12:57:39.352683Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19863188480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150695051264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 550, + "segment_count_current_delta": 0, + "segment_count_peak": 550, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T125740440644Z.pickle", + "timestamp_utc": "2026-08-01T12:57:40.445650Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T125740440644Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T125740440644Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a8309107515e19bd5cb73c9571dc9507ad4f9750 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T125740440644Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebcd7f826e4d671af16d5f990a11dfb7efcb4d7821f7ecc429b902b3caaca5d3 +size 147291 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T125741524351Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T125741524351Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd09450cbdd3089e3ae2dc04a2af9adfd1e05d9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T125741524351Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526842880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133861656576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526842880, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 72526842880, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_peak": 19863188480, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_peak": 150695051264, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T12:57:41.523914Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526842880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115709032448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526842880, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 72526842880, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_peak": 19863188480, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_peak": 150695051264, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T12:57:40.446535Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19863188480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150695051264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 550, + "segment_count_current_delta": 0, + "segment_count_peak": 550, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T125741524351Z.pickle", + "timestamp_utc": "2026-08-01T12:57:41.528782Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T125741524351Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T125741524351Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2730e47451ee0d288b33d71dd1ce35bee45308b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T125741524351Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c4a435960febb5040e80d50456d093a0713468b48242c1217f0b871734c6c40 +size 147291 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T125742611176Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T125742611176Z.json new file mode 100644 index 0000000000000000000000000000000000000000..549269bdc4db506683fd28424f86b8a49332a034 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T125742611176Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526842880, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133861656576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526842880, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 72526842880, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_peak": 19863188480, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_peak": 150695051264, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T12:57:42.610741Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526842880, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115709032448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526842880, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 72526842880, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_peak": 19863188480, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_peak": 150695051264, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T12:57:41.529634Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19863188480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150695051264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 550, + "segment_count_current_delta": 0, + "segment_count_peak": 550, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T125742611176Z.pickle", + "timestamp_utc": "2026-08-01T12:57:42.615806Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T125742611176Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T125742611176Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1f277b6bf64a29059b6fddeacffcae08de4e7837 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T125742611176Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c296f7d767b1ec1891ecabc93d71bab416c21b0c57bebd66c478e388cb7854c +size 147291 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T125841858274Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T125841858274Z.json new file mode 100644 index 0000000000000000000000000000000000000000..173b0bb9b172db1a7fb3edec65f5db602cd5664e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T125841858274Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61320123392, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 133861656576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61320123392, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 61320123392, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 10228411392, + "inactive_split_bytes_peak": 19863188480, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 150764257280, + "reserved_bytes_peak": 150764257280, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T12:58:41.857792Z", + "world_size": 8 + }, + "allocated_bytes_current": 61320123392, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 115709032448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526842880, + "active_bytes_peak": 133861656576, + "allocated_bytes_current": 72526842880, + "allocated_bytes_peak": 115709032448, + "device": 0, + "inactive_split_bytes_current": 19804468224, + "inactive_split_bytes_peak": 19863188480, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 150695051264, + "reserved_bytes_peak": 150695051264, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T12:57:42.616726Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10228411392, + "inactive_split_bytes_current_delta": -9576056832, + "inactive_split_bytes_peak": 19863188480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 150764257280, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 150764257280, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 555, + "segment_count_current_delta": 5, + "segment_count_peak": 555, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T125841858274Z.pickle", + "timestamp_utc": "2026-08-01T12:58:41.942335Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T125841858274Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T125841858274Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fd24da43b0c98004b6dccfe3feb3161898173fa9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T125841858274Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07defdc1d2fa309c00b355b5758ab2a939f425bc20c337f0f8c3f2c596c46aa1 +size 3958476 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T125731533521Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T125731533521Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d35d83388ac6e036d6ad87d45c33f98ccc7f5c86 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T125731533521Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 92058284032, + "active_bytes_peak_delta": 56591947776, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92058284032, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92058284032, + "device": 1, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_peak": 13390509568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 95282003968, + "reserved_bytes_peak": 95282003968, + "segment_count_current": 371, + "segment_count_peak": 371, + "timestamp_utc": "2026-08-01T12:57:31.532916Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 92058284032, + "allocated_bytes_peak_delta": 56591947776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:56:23.361281Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_current_delta": 6011149312, + "inactive_split_bytes_peak": 13390509568, + "inactive_split_bytes_peak_delta": 13230123520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 95282003968, + "reserved_bytes_current_delta": 56037998592, + "reserved_bytes_peak": 95282003968, + "reserved_bytes_peak_delta": 56037998592, + "segment_count_current": 371, + "segment_count_current_delta": 300, + "segment_count_peak": 371, + "segment_count_peak_delta": 300, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T125731533521Z.pickle", + "timestamp_utc": "2026-08-01T12:57:31.650625Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T125731533521Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T125731533521Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0eec3f6f5063cd34ebaa75bd4d3ffd6fcdddd862 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T125731533521Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3df105c9922373a16175df429184b7fd7231b3ac6adc358b3ffbafcfbe98091 +size 5766536 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T125732832998Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T125732832998Z.json new file mode 100644 index 0000000000000000000000000000000000000000..121135675e02af28895d141c7e96df473757c229 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T125732832998Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115695290368, + "active_bytes_peak_delta": 23637006336, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115695290368, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115695290368, + "device": 1, + "inactive_split_bytes_current": 7020577792, + "inactive_split_bytes_peak": 15491064832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:32.832595Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115695290368, + "allocated_bytes_peak_delta": 23637006336, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92058284032, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92058284032, + "device": 1, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_peak": 13390509568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 95282003968, + "reserved_bytes_peak": 95282003968, + "segment_count_current": 371, + "segment_count_peak": 371, + "timestamp_utc": "2026-08-01T12:57:31.671441Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7020577792, + "inactive_split_bytes_current_delta": 876609536, + "inactive_split_bytes_peak": 15491064832, + "inactive_split_bytes_peak_delta": 2100555264, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118669443072, + "reserved_bytes_current_delta": 23387439104, + "reserved_bytes_peak": 118669443072, + "reserved_bytes_peak_delta": 23387439104, + "segment_count_current": 425, + "segment_count_current_delta": 54, + "segment_count_peak": 425, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T125732832998Z.pickle", + "timestamp_utc": "2026-08-01T12:57:32.944276Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T125732832998Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T125732832998Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..02623bdcb866d4296d01c08e6489bc5a8c501d68 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T125732832998Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb87efaa45244362621e6b377fe65c04601d16787bd969cd4ba9b1319c67f335 +size 5891018 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T125734132992Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T125734132992Z.json new file mode 100644 index 0000000000000000000000000000000000000000..185b2325d950759f330feeb1def05ea325d520bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T125734132992Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115705634816, + "active_bytes_peak_delta": 10344448, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115705634816, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 8494875648, + "inactive_split_bytes_peak": 15550785536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:34.132597Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 10344448, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115695290368, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115695290368, + "device": 1, + "inactive_split_bytes_current": 7020577792, + "inactive_split_bytes_peak": 15491064832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:32.960075Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8494875648, + "inactive_split_bytes_current_delta": 1474297856, + "inactive_split_bytes_peak": 15550785536, + "inactive_split_bytes_peak_delta": 59720704, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118669443072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118669443072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 425, + "segment_count_current_delta": 0, + "segment_count_peak": 425, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T125734132992Z.pickle", + "timestamp_utc": "2026-08-01T12:57:34.243919Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T125734132992Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T125734132992Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..792f8c7ac3c613e540d00a839c9554db662a158f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T125734132992Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fda80861e2c7a20b8abe7780eb66b55ab708389c3b6d94ed47e0caa6652abc8d +size 5890180 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T125737022158Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T125737022158Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5cc00cd8e1367ee15dad3bb4bb4ba1d5ca782e96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T125737022158Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 18153318400, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 150455975936, + "reserved_bytes_peak": 150455975936, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:37.021598Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115705634816, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 8494875648, + "inactive_split_bytes_peak": 15550785536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:34.259571Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 10659557376, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 4989324288, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 150455975936, + "reserved_bytes_current_delta": 31786532864, + "reserved_bytes_peak": 150455975936, + "reserved_bytes_peak_delta": 31786532864, + "segment_count_current": 548, + "segment_count_current_delta": 123, + "segment_count_peak": 548, + "segment_count_peak_delta": 123, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T125737022158Z.pickle", + "timestamp_utc": "2026-08-01T12:57:37.146809Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T125737022158Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T125737022158Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..794fdd8f99140fda06b3716b933de62d28d081ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T125737022158Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f1d55fe86f13d5ea52040c674d9ac0b1d32eff5256ed104ca3f1ce0be350c08 +size 6153211 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T125738257561Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T125738257561Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2bd9f91ca4d6079667c47625696086992c104eca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T125738257561Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 150455975936, + "reserved_bytes_peak": 150455975936, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:38.256620Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 150455975936, + "reserved_bytes_peak": 150455975936, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:37.164365Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 150455975936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150455975936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 548, + "segment_count_current_delta": 0, + "segment_count_peak": 548, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T125738257561Z.pickle", + "timestamp_utc": "2026-08-01T12:57:38.261873Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T125738257561Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T125738257561Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..52906ba21b4f0a4a6d7ea2ed6037b2a716a38cab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T125738257561Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:734500286586b479ae5306882abae247bf8d2840e34d034d6fbdf55714b77fa4 +size 146270 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T125739347529Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T125739347529Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c61db80af1b1616135ba031ddd95d99952bbc2b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T125739347529Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 150455975936, + "reserved_bytes_peak": 150455975936, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:39.347072Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 150455975936, + "reserved_bytes_peak": 150455975936, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:38.262780Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 150455975936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150455975936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 548, + "segment_count_current_delta": 0, + "segment_count_peak": 548, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T125739347529Z.pickle", + "timestamp_utc": "2026-08-01T12:57:39.352316Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T125739347529Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T125739347529Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..159634c051c0ab829c4f68b2759d68939ac101ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T125739347529Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7da6d016637b4186be24d9b4b7fdd9b56882b32fa769627c97549852fdf215e9 +size 146301 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T125740440565Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T125740440565Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a223e043756a344da57df431097290951aeef691 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T125740440565Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 150455975936, + "reserved_bytes_peak": 150455975936, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:40.440131Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 150455975936, + "reserved_bytes_peak": 150455975936, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:39.353186Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 150455975936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150455975936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 548, + "segment_count_current_delta": 0, + "segment_count_peak": 548, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T125740440565Z.pickle", + "timestamp_utc": "2026-08-01T12:57:40.444937Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T125740440565Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T125740440565Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0593f6bb81bf3a2bcafc1c0c81bdd58f8596295a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T125740440565Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4bcb593c7497601b6307451e0587e5031f291eba09fff71783939b33262c630 +size 146270 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T125741524532Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T125741524532Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3242db7daa4909bb84973125b147af332cafeb41 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T125741524532Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 150455975936, + "reserved_bytes_peak": 150455975936, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:41.524092Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 150455975936, + "reserved_bytes_peak": 150455975936, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:40.445786Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 150455975936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150455975936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 548, + "segment_count_current_delta": 0, + "segment_count_peak": 548, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T125741524532Z.pickle", + "timestamp_utc": "2026-08-01T12:57:41.530143Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T125741524532Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T125741524532Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e9eaa709495577534353e3136275fe4971545ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T125741524532Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5199c7f6488d46e49238e93cac840ecfdcc2519eacb409a4f80f23fcb94bd663 +size 146301 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T125742611187Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T125742611187Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bccb7f808fb1f80f640081b5765575c2a0e58198 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T125742611187Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 150455975936, + "reserved_bytes_peak": 150455975936, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:42.610761Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 150455975936, + "reserved_bytes_peak": 150455975936, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:41.531035Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 150455975936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150455975936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 548, + "segment_count_current_delta": 0, + "segment_count_peak": 548, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T125742611187Z.pickle", + "timestamp_utc": "2026-08-01T12:57:42.615850Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T125742611187Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T125742611187Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..344b55a66bae78eda3b2f0753bdbd3056917048d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T125742611187Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b00d76839a0ba185f54f2e3f5b9bfd1433283e20c80a8ad0d5e07f8b347381b +size 146270 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T125841858293Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T125841858293Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a29d179ac0d35b740b084ec313a03dd90bb98d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T125841858293Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61317944320, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61317944320, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 61317944320, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 11281263616, + "inactive_split_bytes_peak": 20540109824, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 150525181952, + "reserved_bytes_peak": 150525181952, + "segment_count_current": 553, + "segment_count_peak": 553, + "timestamp_utc": "2026-08-01T12:58:41.857773Z", + "world_size": 8 + }, + "allocated_bytes_current": 61317944320, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 1, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 150455975936, + "reserved_bytes_peak": 150455975936, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:42.616766Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11281263616, + "inactive_split_bytes_current_delta": -7873169408, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 150525181952, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 150525181952, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 553, + "segment_count_current_delta": 5, + "segment_count_peak": 553, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T125841858293Z.pickle", + "timestamp_utc": "2026-08-01T12:58:41.942287Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T125841858293Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T125841858293Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d586c6f9703208dd83629ab39fcc91b86b7d0c5e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T125841858293Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26634f0bb3151f58abc3c9fd639d0416319e11b386cb7dadc2e536e99f15a31d +size 3941224 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T125731533603Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T125731533603Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5bd6e5ecd0af8a0bcf1fad1de1b154b7029f027f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T125731533603Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 92058284032, + "active_bytes_peak_delta": 56591947776, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92058284032, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92058284032, + "device": 2, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_peak": 13390509568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 95282003968, + "reserved_bytes_peak": 95282003968, + "segment_count_current": 371, + "segment_count_peak": 371, + "timestamp_utc": "2026-08-01T12:57:31.532955Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 92058284032, + "allocated_bytes_peak_delta": 56591947776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:56:23.360778Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_current_delta": 6011149312, + "inactive_split_bytes_peak": 13390509568, + "inactive_split_bytes_peak_delta": 13230123520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 95282003968, + "reserved_bytes_current_delta": 56037998592, + "reserved_bytes_peak": 95282003968, + "reserved_bytes_peak_delta": 56037998592, + "segment_count_current": 371, + "segment_count_current_delta": 300, + "segment_count_peak": 371, + "segment_count_peak_delta": 300, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T125731533603Z.pickle", + "timestamp_utc": "2026-08-01T12:57:31.651466Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T125731533603Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T125731533603Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e858fb5585596161de95ad96da6e7d77a17a83b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T125731533603Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26aef30349969d1e17410d8106a99ad1b7119884468d03368044c2fd2ef6a9cc +size 5766495 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T125732833000Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T125732833000Z.json new file mode 100644 index 0000000000000000000000000000000000000000..881db4acbd302b54e15c1a0816543d40bdceef70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T125732833000Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115695290368, + "active_bytes_peak_delta": 23637006336, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115695290368, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115695290368, + "device": 2, + "inactive_split_bytes_current": 7020577792, + "inactive_split_bytes_peak": 15493161984, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:32.832586Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115695290368, + "allocated_bytes_peak_delta": 23637006336, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92058284032, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92058284032, + "device": 2, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_peak": 13390509568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 95282003968, + "reserved_bytes_peak": 95282003968, + "segment_count_current": 371, + "segment_count_peak": 371, + "timestamp_utc": "2026-08-01T12:57:31.672127Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7020577792, + "inactive_split_bytes_current_delta": 876609536, + "inactive_split_bytes_peak": 15493161984, + "inactive_split_bytes_peak_delta": 2102652416, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118669443072, + "reserved_bytes_current_delta": 23387439104, + "reserved_bytes_peak": 118669443072, + "reserved_bytes_peak_delta": 23387439104, + "segment_count_current": 425, + "segment_count_current_delta": 54, + "segment_count_peak": 425, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T125732833000Z.pickle", + "timestamp_utc": "2026-08-01T12:57:32.940362Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T125732833000Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T125732833000Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eb776b2fb187ede790b3a85e7f184bbc12e1575e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T125732833000Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53b54a8d5edcc138e54deccd27810ab049603e2a470096634895756b9b3f8538 +size 5890978 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T125734133015Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T125734133015Z.json new file mode 100644 index 0000000000000000000000000000000000000000..83d32703ed8073f4cb10ad32b658ecd1c7325888 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T125734133015Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115705634816, + "active_bytes_peak_delta": 10344448, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115705634816, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 8496972800, + "inactive_split_bytes_peak": 15550785536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:34.132589Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 10344448, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115695290368, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115695290368, + "device": 2, + "inactive_split_bytes_current": 7020577792, + "inactive_split_bytes_peak": 15493161984, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:32.954512Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8496972800, + "inactive_split_bytes_current_delta": 1476395008, + "inactive_split_bytes_peak": 15550785536, + "inactive_split_bytes_peak_delta": 57623552, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118669443072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118669443072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 425, + "segment_count_current_delta": 0, + "segment_count_peak": 425, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T125734133015Z.pickle", + "timestamp_utc": "2026-08-01T12:57:34.238239Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T125734133015Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T125734133015Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..abb47ddfdbbba4cf41b383863c18ef772ab8eec6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T125734133015Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dae63706a05d164af496a84c322615fcd33db476a1b21f6e8fa55a3095222ee +size 5890181 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T125737022094Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T125737022094Z.json new file mode 100644 index 0000000000000000000000000000000000000000..397860f2b60df2ccaadab3093f7bc5dc7a9f7226 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T125737022094Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 18153318400, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_peak": 20544304128, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 150458073088, + "reserved_bytes_peak": 150458073088, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T12:57:37.021551Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115705634816, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 8496972800, + "inactive_split_bytes_peak": 15550785536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:34.251363Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_current_delta": 10661654528, + "inactive_split_bytes_peak": 20544304128, + "inactive_split_bytes_peak_delta": 4993518592, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 150458073088, + "reserved_bytes_current_delta": 31788630016, + "reserved_bytes_peak": 150458073088, + "reserved_bytes_peak_delta": 31788630016, + "segment_count_current": 549, + "segment_count_current_delta": 124, + "segment_count_peak": 549, + "segment_count_peak_delta": 124, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T125737022094Z.pickle", + "timestamp_utc": "2026-08-01T12:57:37.142499Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T125737022094Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T125737022094Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e2afdb34657bdd5a66ddc8954c3468de76ff6517 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T125737022094Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76b3fed83de825254cde70ac2fff7f36b02d60e0ebf90f821c7bca7789335732 +size 6153297 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T125738257526Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T125738257526Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c725549c9d523df2d219939505a8d7ba7b532305 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T125738257526Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_peak": 20544304128, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 150458073088, + "reserved_bytes_peak": 150458073088, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T12:57:38.256631Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_peak": 20544304128, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 150458073088, + "reserved_bytes_peak": 150458073088, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T12:57:37.157454Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20544304128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 150458073088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150458073088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 549, + "segment_count_current_delta": 0, + "segment_count_peak": 549, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T125738257526Z.pickle", + "timestamp_utc": "2026-08-01T12:57:38.261813Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T125738257526Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T125738257526Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0121a2efe18df755374b5c5e62caf22629b3b2be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T125738257526Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2d0142b31ac638fe8ec2f8749a629beb71ba5da218d2494dcee36ae13920cb9 +size 146439 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T125739347435Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T125739347435Z.json new file mode 100644 index 0000000000000000000000000000000000000000..727039e173dceff0c1b017333ade3af3309cab7f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T125739347435Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_peak": 20544304128, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 150458073088, + "reserved_bytes_peak": 150458073088, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T12:57:39.346998Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_peak": 20544304128, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 150458073088, + "reserved_bytes_peak": 150458073088, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T12:57:38.262729Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20544304128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 150458073088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150458073088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 549, + "segment_count_current_delta": 0, + "segment_count_peak": 549, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T125739347435Z.pickle", + "timestamp_utc": "2026-08-01T12:57:39.351777Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T125739347435Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T125739347435Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f2cf196d58d3a2b77fc6f9ffa80a9bf59df7c79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T125739347435Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac9e880539a6acc99c8a96d994dfcdc9a65359313bda86fdcb7763ee0643ac32 +size 146439 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T125740440510Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T125740440510Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd7bee5e1a551fa0b3a23b5a656d23f936244bad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T125740440510Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_peak": 20544304128, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 150458073088, + "reserved_bytes_peak": 150458073088, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T12:57:40.440100Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_peak": 20544304128, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 150458073088, + "reserved_bytes_peak": 150458073088, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T12:57:39.352656Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20544304128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 150458073088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150458073088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 549, + "segment_count_current_delta": 0, + "segment_count_peak": 549, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T125740440510Z.pickle", + "timestamp_utc": "2026-08-01T12:57:40.444724Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T125740440510Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T125740440510Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2c74b8f8312b839aff4e34ff3dd6d35dafacd9e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T125740440510Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:347c6337f23130591ad9777fb695fcd72b31462a8b9f7ea95e4272535d9c5825 +size 146439 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T125741524302Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T125741524302Z.json new file mode 100644 index 0000000000000000000000000000000000000000..22c5ba372f617d6bfeea427917f57e949c5e42c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T125741524302Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_peak": 20544304128, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 150458073088, + "reserved_bytes_peak": 150458073088, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T12:57:41.523894Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_peak": 20544304128, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 150458073088, + "reserved_bytes_peak": 150458073088, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T12:57:40.445622Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20544304128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 150458073088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150458073088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 549, + "segment_count_current_delta": 0, + "segment_count_peak": 549, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T125741524302Z.pickle", + "timestamp_utc": "2026-08-01T12:57:41.528469Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T125741524302Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T125741524302Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8b7801c267b0c9fcba88cd50ab928d3aaa289b4c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T125741524302Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4440e8600b906d41a943589e8b2b80b32a105f0fc4765968c3ed61f004cf96c3 +size 146439 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T125742611283Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T125742611283Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34926a2aa359ba3b3004e2b7ba0b8268ee54ad16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T125742611283Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_peak": 20544304128, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 150458073088, + "reserved_bytes_peak": 150458073088, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T12:57:42.610845Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_peak": 20544304128, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 150458073088, + "reserved_bytes_peak": 150458073088, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T12:57:41.529339Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20544304128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 150458073088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150458073088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 549, + "segment_count_current_delta": 0, + "segment_count_peak": 549, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T125742611283Z.pickle", + "timestamp_utc": "2026-08-01T12:57:42.616195Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T125742611283Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T125742611283Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dac4f2c3cba30a3a44218f5c0b32b96ccd8fe0d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T125742611283Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d937f238f9bae3b16c058a5fbf21d567261457ec35d18771f97450c8d2ed65db +size 146439 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T125841858563Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T125841858563Z.json new file mode 100644 index 0000000000000000000000000000000000000000..25a8c7786650c33c545f36ab3d390e343233f7d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T125841858563Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61317944320, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61317944320, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 61317944320, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 11285457920, + "inactive_split_bytes_peak": 20544304128, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 150527279104, + "reserved_bytes_peak": 150527279104, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T12:58:41.857990Z", + "world_size": 8 + }, + "allocated_bytes_current": 61317944320, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 2, + "inactive_split_bytes_current": 19158627328, + "inactive_split_bytes_peak": 20544304128, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 150458073088, + "reserved_bytes_peak": 150458073088, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T12:57:42.617100Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 11285457920, + "inactive_split_bytes_current_delta": -7873169408, + "inactive_split_bytes_peak": 20544304128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 150527279104, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 150527279104, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 554, + "segment_count_current_delta": 5, + "segment_count_peak": 554, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T125841858563Z.pickle", + "timestamp_utc": "2026-08-01T12:58:41.941714Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T125841858563Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T125841858563Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ad2c441c2cbe9a332ef68dd1d0f65eac9e8d3dd7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T125841858563Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c45521cd15f7422c0b5861ea68be3d4caf026d7d0c3fd81ceb7a677624e6073c +size 3941295 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T125731533561Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T125731533561Z.json new file mode 100644 index 0000000000000000000000000000000000000000..761973963bb378b13f5fa07132e6c0ea9db529d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T125731533561Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 92058284032, + "active_bytes_peak_delta": 56591947776, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92058284032, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92058284032, + "device": 3, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_peak": 13390509568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 95282003968, + "reserved_bytes_peak": 95282003968, + "segment_count_current": 371, + "segment_count_peak": 371, + "timestamp_utc": "2026-08-01T12:57:31.532941Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 92058284032, + "allocated_bytes_peak_delta": 56591947776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:56:23.378959Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_current_delta": 6011149312, + "inactive_split_bytes_peak": 13390509568, + "inactive_split_bytes_peak_delta": 13230123520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 95282003968, + "reserved_bytes_current_delta": 56037998592, + "reserved_bytes_peak": 95282003968, + "reserved_bytes_peak_delta": 56037998592, + "segment_count_current": 371, + "segment_count_current_delta": 300, + "segment_count_peak": 371, + "segment_count_peak_delta": 300, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T125731533561Z.pickle", + "timestamp_utc": "2026-08-01T12:57:31.648590Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T125731533561Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T125731533561Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..61a29ca7bcd8cb81110be29b4bb0b3c9a8745dc0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T125731533561Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b5888ea74aae31e76dce6e49f8f7bc678b3158bb8acd4aa373d267ce2049cb6 +size 5766542 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T125732832986Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T125732832986Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9f00c3c15d03acf3a89222e953c6dcfa2ecb20d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T125732832986Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115695290368, + "active_bytes_peak_delta": 23637006336, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115695290368, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115695290368, + "device": 3, + "inactive_split_bytes_current": 7024772096, + "inactive_split_bytes_peak": 15493161984, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:32.832573Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115695290368, + "allocated_bytes_peak_delta": 23637006336, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92058284032, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92058284032, + "device": 3, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_peak": 13390509568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 95282003968, + "reserved_bytes_peak": 95282003968, + "segment_count_current": 371, + "segment_count_peak": 371, + "timestamp_utc": "2026-08-01T12:57:31.665720Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7024772096, + "inactive_split_bytes_current_delta": 880803840, + "inactive_split_bytes_peak": 15493161984, + "inactive_split_bytes_peak_delta": 2102652416, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118669443072, + "reserved_bytes_current_delta": 23387439104, + "reserved_bytes_peak": 118669443072, + "reserved_bytes_peak_delta": 23387439104, + "segment_count_current": 425, + "segment_count_current_delta": 54, + "segment_count_peak": 425, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T125732832986Z.pickle", + "timestamp_utc": "2026-08-01T12:57:32.936196Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T125732832986Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T125732832986Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cd3cc67123050a7bf2e94a7e18004441658621ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T125732832986Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c3a72ceb374441ffc7961fa8fddf109f560bedb69edb382eaefc80dcf29d7c8 +size 5891018 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T125734132998Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T125734132998Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5689e65b8668c4931c98c537af3622031cb1d6a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T125734132998Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115705634816, + "active_bytes_peak_delta": 10344448, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115705634816, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 8490681344, + "inactive_split_bytes_peak": 15550785536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:34.132594Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 10344448, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115695290368, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115695290368, + "device": 3, + "inactive_split_bytes_current": 7024772096, + "inactive_split_bytes_peak": 15493161984, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:32.948938Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8490681344, + "inactive_split_bytes_current_delta": 1465909248, + "inactive_split_bytes_peak": 15550785536, + "inactive_split_bytes_peak_delta": 57623552, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118669443072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118669443072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 425, + "segment_count_current_delta": 0, + "segment_count_peak": 425, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T125734132998Z.pickle", + "timestamp_utc": "2026-08-01T12:57:34.238941Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T125734132998Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T125734132998Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..374567307994f4bab18b08f10013aaa2fbaf8f74 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T125734132998Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8530ecc095d273f16d9b4e3d9683de8d3c59d2ad3eb612cf859e92b5a2b79c5 +size 5890168 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T125737022084Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T125737022084Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc1091e99d27fdcf566258079eb7727a002930ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T125737022084Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 133855021056, + "active_bytes_peak_delta": 18149386240, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_peak": 19783947264, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 150200123392, + "reserved_bytes_peak": 150200123392, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:37.021544Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115705634816, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 8490681344, + "inactive_split_bytes_peak": 15550785536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:34.251075Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_current_delta": 9491443712, + "inactive_split_bytes_peak": 19783947264, + "inactive_split_bytes_peak_delta": 4233161728, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 150200123392, + "reserved_bytes_current_delta": 31530680320, + "reserved_bytes_peak": 150200123392, + "reserved_bytes_peak_delta": 31530680320, + "segment_count_current": 548, + "segment_count_current_delta": 123, + "segment_count_peak": 548, + "segment_count_peak_delta": 123, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T125737022084Z.pickle", + "timestamp_utc": "2026-08-01T12:57:37.140593Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T125737022084Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T125737022084Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..898039539b54f9ec1d9c174792bcc21893e87740 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T125737022084Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:190ab647dc6a42f8ee506bfa432064df3afbfce0c93c1b79f68d45b2354c12e7 +size 6135229 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T125738257505Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T125738257505Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f628f67f4ab0fc665e99cc76da9c3cccd517e1fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T125738257505Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133855021056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_peak": 19783947264, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 150200123392, + "reserved_bytes_peak": 150200123392, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:38.256647Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_peak": 19783947264, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 150200123392, + "reserved_bytes_peak": 150200123392, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:37.154444Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19783947264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 150200123392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150200123392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 548, + "segment_count_current_delta": 0, + "segment_count_peak": 548, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T125738257505Z.pickle", + "timestamp_utc": "2026-08-01T12:57:38.261578Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T125738257505Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T125738257505Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8671bcd8071d8202f213248044ef81117ce6ac7a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T125738257505Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0fe746b48abc44a080591ab55cb32c857ed2c4ee2bae4e0214774db97b3fc95 +size 145155 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T125739347433Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T125739347433Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee3f41914360d97ba6a33286ecf1be92b2b657b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T125739347433Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133855021056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_peak": 19783947264, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 150200123392, + "reserved_bytes_peak": 150200123392, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:39.346993Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_peak": 19783947264, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 150200123392, + "reserved_bytes_peak": 150200123392, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:38.262481Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19783947264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 150200123392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150200123392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 548, + "segment_count_current_delta": 0, + "segment_count_peak": 548, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T125739347433Z.pickle", + "timestamp_utc": "2026-08-01T12:57:39.351621Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T125739347433Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T125739347433Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3eb080cc3d501dad29146d5ac2f97f19140a7fe5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T125739347433Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d237398452f558e1b6083884ba6c8935a24b60634940df760c3195dd90f0c813 +size 145155 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T125740440562Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T125740440562Z.json new file mode 100644 index 0000000000000000000000000000000000000000..828eb62dcb745e2bf908a3c5f46278e17594ddd3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T125740440562Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133855021056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_peak": 19783947264, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 150200123392, + "reserved_bytes_peak": 150200123392, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:40.440117Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_peak": 19783947264, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 150200123392, + "reserved_bytes_peak": 150200123392, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:39.352504Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19783947264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 150200123392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150200123392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 548, + "segment_count_current_delta": 0, + "segment_count_peak": 548, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T125740440562Z.pickle", + "timestamp_utc": "2026-08-01T12:57:40.445093Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T125740440562Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T125740440562Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72159c943672e9c1e7fa857c044e0360fecab994 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T125740440562Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d071edadc7ce78ed0475cbda301c7acf4eb8d056a08d1cb622e5ea2c9c27b3b4 +size 145155 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T125741524347Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T125741524347Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2fb1a0d6d126753c748dbf56b05b6a21ed5ff58b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T125741524347Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133855021056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_peak": 19783947264, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 150200123392, + "reserved_bytes_peak": 150200123392, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:41.523915Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_peak": 19783947264, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 150200123392, + "reserved_bytes_peak": 150200123392, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:40.445969Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19783947264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 150200123392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150200123392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 548, + "segment_count_current_delta": 0, + "segment_count_peak": 548, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T125741524347Z.pickle", + "timestamp_utc": "2026-08-01T12:57:41.528729Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T125741524347Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T125741524347Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a26bd8297271508bffaf60f1240fd7b56f7a4803 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T125741524347Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5143422bb72a6be5f68bbb37e096e45c66338c1bed6f9ee9b84ee73f3aa4bf32 +size 145155 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T125742611197Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T125742611197Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f60ff33a2d00486460735a0d2ee0b310d057ab58 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T125742611197Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133855021056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_peak": 19783947264, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 150200123392, + "reserved_bytes_peak": 150200123392, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:42.610755Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_peak": 19783947264, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 150200123392, + "reserved_bytes_peak": 150200123392, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:41.529594Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 19783947264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 150200123392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150200123392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 548, + "segment_count_current_delta": 0, + "segment_count_peak": 548, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T125742611197Z.pickle", + "timestamp_utc": "2026-08-01T12:57:42.615635Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T125742611197Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T125742611197Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8f533b1107e22dae8fe14b77edd850d2bf24cc35 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T125742611197Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff98d0af1de3c9aa2150ce528614b7224c61c4bbe348b35e2d02f5f9518d0ac3 +size 145155 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T125841858350Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T125841858350Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a673658547e6ac9ba92fc568b1393c5d3fd2ebb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T125841858350Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61317944320, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 133855021056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61317944320, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 61317944320, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 11283360768, + "inactive_split_bytes_peak": 19783947264, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 150269329408, + "reserved_bytes_peak": 150269329408, + "segment_count_current": 553, + "segment_count_peak": 553, + "timestamp_utc": "2026-08-01T12:58:41.857780Z", + "world_size": 8 + }, + "allocated_bytes_current": 61317944320, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133855021056, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 3, + "inactive_split_bytes_current": 17982125056, + "inactive_split_bytes_peak": 19783947264, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 150200123392, + "reserved_bytes_peak": 150200123392, + "segment_count_current": 548, + "segment_count_peak": 548, + "timestamp_utc": "2026-08-01T12:57:42.616562Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 11283360768, + "inactive_split_bytes_current_delta": -6698764288, + "inactive_split_bytes_peak": 19783947264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 150269329408, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 150269329408, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 553, + "segment_count_current_delta": 5, + "segment_count_peak": 553, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T125841858350Z.pickle", + "timestamp_utc": "2026-08-01T12:58:41.939411Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T125841858350Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T125841858350Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e5a8768428aa7a720421623c2244f199fb25e2a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T125841858350Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7d3aceaaf3cea3e46ea11095563d443a6b12b51fe2f1008142f0aaefa00d37b +size 3940803 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T125731533521Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T125731533521Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31fd4e5d96470e057bc1742a0b90977e155b3813 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T125731533521Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508975104, + "active_bytes_current_delta": 37277077504, + "active_bytes_peak": 92053434368, + "active_bytes_peak_delta": 56586655744, + "after": { + "active_bytes_current": 72508975104, + "active_bytes_peak": 92053434368, + "allocated_bytes_current": 72508975104, + "allocated_bytes_peak": 92053434368, + "device": 4, + "inactive_split_bytes_current": 5851109376, + "inactive_split_bytes_peak": 12177366528, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 94812241920, + "reserved_bytes_peak": 94812241920, + "segment_count_current": 370, + "segment_count_peak": 370, + "timestamp_utc": "2026-08-01T12:57:31.532939Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508975104, + "allocated_bytes_current_delta": 37277077504, + "allocated_bytes_peak": 92053434368, + "allocated_bytes_peak_delta": 56586655744, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:56:23.360613Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5851109376, + "inactive_split_bytes_current_delta": 5718732800, + "inactive_split_bytes_peak": 12177366528, + "inactive_split_bytes_peak_delta": 12023956992, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 94812241920, + "reserved_bytes_current_delta": 55568236544, + "reserved_bytes_peak": 94812241920, + "reserved_bytes_peak_delta": 55568236544, + "segment_count_current": 370, + "segment_count_current_delta": 299, + "segment_count_peak": 370, + "segment_count_peak_delta": 299, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T125731533521Z.pickle", + "timestamp_utc": "2026-08-01T12:57:31.652549Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T125731533521Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T125731533521Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..269360cda77d6924510d7c00baf9362566f1d65c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T125731533521Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:466af516d6ddaca0ffd4d734d30b10d449205521317cb34665dcca2c403a8c8f +size 5774990 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T125732832959Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T125732832959Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd118ad070809a064435138729169bd481f72258 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T125732832959Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508975104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115702319104, + "active_bytes_peak_delta": 23648884736, + "after": { + "active_bytes_current": 72508975104, + "active_bytes_peak": 115702319104, + "allocated_bytes_current": 72508975104, + "allocated_bytes_peak": 115702319104, + "device": 4, + "inactive_split_bytes_current": 9374324736, + "inactive_split_bytes_peak": 14869823488, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 118451339264, + "reserved_bytes_peak": 118451339264, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T12:57:32.832547Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508975104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115702319104, + "allocated_bytes_peak_delta": 23648884736, + "before": { + "active_bytes_current": 72508975104, + "active_bytes_peak": 92053434368, + "allocated_bytes_current": 72508975104, + "allocated_bytes_peak": 92053434368, + "device": 4, + "inactive_split_bytes_current": 5851109376, + "inactive_split_bytes_peak": 12177366528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 94812241920, + "reserved_bytes_peak": 94812241920, + "segment_count_current": 370, + "segment_count_peak": 370, + "timestamp_utc": "2026-08-01T12:57:31.673033Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9374324736, + "inactive_split_bytes_current_delta": 3523215360, + "inactive_split_bytes_peak": 14869823488, + "inactive_split_bytes_peak_delta": 2692456960, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 118451339264, + "reserved_bytes_current_delta": 23639097344, + "reserved_bytes_peak": 118451339264, + "reserved_bytes_peak_delta": 23639097344, + "segment_count_current": 422, + "segment_count_current_delta": 52, + "segment_count_peak": 422, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T125732832959Z.pickle", + "timestamp_utc": "2026-08-01T12:57:32.942256Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T125732832959Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T125732832959Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dae762c1705021cfeaf28d776a782a0f4f1fadcc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T125732832959Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fb352b4d5c36ad78e734664fac25033339cfff8e2f50b027e86c79c8e3c1fa3 +size 5901281 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T125734132996Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T125734132996Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e194312d0bb344eb33481f15cf2cdaa950455c18 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T125734132996Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508975104, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115704547328, + "active_bytes_peak_delta": 2228224, + "after": { + "active_bytes_current": 72508975104, + "active_bytes_peak": 115704547328, + "allocated_bytes_current": 72508975104, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 6731913216, + "inactive_split_bytes_peak": 14869823488, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 118451339264, + "reserved_bytes_peak": 118451339264, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T12:57:34.132586Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508975104, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115704547328, + "allocated_bytes_peak_delta": 2228224, + "before": { + "active_bytes_current": 72508975104, + "active_bytes_peak": 115702319104, + "allocated_bytes_current": 72508975104, + "allocated_bytes_peak": 115702319104, + "device": 4, + "inactive_split_bytes_current": 9374324736, + "inactive_split_bytes_peak": 14869823488, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 118451339264, + "reserved_bytes_peak": 118451339264, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T12:57:32.956563Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6731913216, + "inactive_split_bytes_current_delta": -2642411520, + "inactive_split_bytes_peak": 14869823488, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 118451339264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118451339264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T125734132996Z.pickle", + "timestamp_utc": "2026-08-01T12:57:34.240103Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T125734132996Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T125734132996Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..054da1a0786ccbf56f644cc7858288525b787da0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T125734132996Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67a136d3086781de6911efcdcaee27ca4caf4f1191a55dc5591a019cd450473c +size 5983242 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T125737022068Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T125737022068Z.json new file mode 100644 index 0000000000000000000000000000000000000000..070f521be859aa6ac654e2b250ea1a58bbd1fbb2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T125737022068Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526018560, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 133855589376, + "active_bytes_peak_delta": 18151042048, + "after": { + "active_bytes_current": 72526018560, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 72526018560, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_peak": 21257594880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148602093568, + "reserved_bytes_peak": 148602093568, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:57:37.021538Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526018560, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115704547328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72508975104, + "active_bytes_peak": 115704547328, + "allocated_bytes_current": 72508975104, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 6731913216, + "inactive_split_bytes_peak": 14869823488, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 118451339264, + "reserved_bytes_peak": 118451339264, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T12:57:34.253565Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_current_delta": 12012220416, + "inactive_split_bytes_peak": 21257594880, + "inactive_split_bytes_peak_delta": 6387771392, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148602093568, + "reserved_bytes_current_delta": 30150754304, + "reserved_bytes_peak": 148602093568, + "reserved_bytes_peak_delta": 30150754304, + "segment_count_current": 545, + "segment_count_current_delta": 123, + "segment_count_peak": 545, + "segment_count_peak_delta": 123, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T125737022068Z.pickle", + "timestamp_utc": "2026-08-01T12:57:37.145973Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T125737022068Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T125737022068Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2de440a3833d3571b1932b1207e66790189688d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T125737022068Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2957bc7a89af614aa5382164761213b08539f4e0689f563bf4ed853028a5f91a +size 6142773 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T125738257425Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T125738257425Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ca62efb4c1ab35a7ddec8c9e632b35bda2d1f05 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T125738257425Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526018560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133855589376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526018560, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 72526018560, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_peak": 21257594880, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148602093568, + "reserved_bytes_peak": 148602093568, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:57:38.256619Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526018560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115704547328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526018560, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 72526018560, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_peak": 21257594880, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148602093568, + "reserved_bytes_peak": 148602093568, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:57:37.161181Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21257594880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148602093568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148602093568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T125738257425Z.pickle", + "timestamp_utc": "2026-08-01T12:57:38.261814Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T125738257425Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T125738257425Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf674f130c379e81c8ec38d5a09998b42a9d06bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T125738257425Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03b05b99ed15ca68ab298ba6d428be4038cef34ed921c96b190373f234dcba92 +size 152359 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T125739347550Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T125739347550Z.json new file mode 100644 index 0000000000000000000000000000000000000000..931b2163561a650579f3c15cfbbfab53bdaf50f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T125739347550Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526018560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133855589376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526018560, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 72526018560, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_peak": 21257594880, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148602093568, + "reserved_bytes_peak": 148602093568, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:57:39.347097Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526018560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115704547328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526018560, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 72526018560, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_peak": 21257594880, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148602093568, + "reserved_bytes_peak": 148602093568, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:57:38.262693Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21257594880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148602093568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148602093568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T125739347550Z.pickle", + "timestamp_utc": "2026-08-01T12:57:39.353196Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T125739347550Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T125739347550Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..21f761935d301b7e9c85343a3417f1a0a63b0fed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T125739347550Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9631cd0a74377e5771945c48f7ceb0e9dbdba775dd62fb58544d224ef042a7f +size 152359 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T125740440474Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T125740440474Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d4459c01ad77376600c07ad33bc046644338f78a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T125740440474Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526018560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133855589376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526018560, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 72526018560, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_peak": 21257594880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148602093568, + "reserved_bytes_peak": 148602093568, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:57:40.440070Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526018560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115704547328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526018560, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 72526018560, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_peak": 21257594880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148602093568, + "reserved_bytes_peak": 148602093568, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:57:39.354059Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21257594880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148602093568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148602093568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T125740440474Z.pickle", + "timestamp_utc": "2026-08-01T12:57:40.444767Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T125740440474Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T125740440474Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..be4a6814452bdaa9989d85b5f516e0bcf50c6c04 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T125740440474Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e24e50b1c5d8b6e4650b8098be280e6f912c23af1c34ff1d6bd7fb5267966892 +size 152359 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T125741524248Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T125741524248Z.json new file mode 100644 index 0000000000000000000000000000000000000000..251a8a6049252ca3aa7731cf62c3d9a295db6162 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T125741524248Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526018560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133855589376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526018560, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 72526018560, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_peak": 21257594880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148602093568, + "reserved_bytes_peak": 148602093568, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:57:41.523847Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526018560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115704547328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526018560, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 72526018560, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_peak": 21257594880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148602093568, + "reserved_bytes_peak": 148602093568, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:57:40.445657Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21257594880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148602093568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148602093568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T125741524248Z.pickle", + "timestamp_utc": "2026-08-01T12:57:41.528441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T125741524248Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T125741524248Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44d2e9e3aae89267fb113833db72b64e801d9293 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T125741524248Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8a4d5e91237ade80b66789f981f879439cee679310ddf465401b123adfd3fc3 +size 152359 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T125742611124Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T125742611124Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ff147232ec4d95aedb3b699dd6409dc49bfb765a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T125742611124Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72526018560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133855589376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72526018560, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 72526018560, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_peak": 21257594880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 148602093568, + "reserved_bytes_peak": 148602093568, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:57:42.610713Z", + "world_size": 8 + }, + "allocated_bytes_current": 72526018560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115704547328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526018560, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 72526018560, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_peak": 21257594880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148602093568, + "reserved_bytes_peak": 148602093568, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:57:41.529318Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 21257594880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 148602093568, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 148602093568, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T125742611124Z.pickle", + "timestamp_utc": "2026-08-01T12:57:42.615572Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T125742611124Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T125742611124Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a9fa2181b321f1f343e000814d288bf8123497f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T125742611124Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:703b0da618d8739fb82e601246abfaecc0bed9fcd327dcbb0859fbdb1b935d48 +size 152359 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T125841858294Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T125841858294Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a2d30ae6e6ef6075fb020ef2dc66fe539e073f60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T125841858294Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61319299072, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 133855589376, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61319299072, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 61319299072, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 9813999616, + "inactive_split_bytes_peak": 21257594880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 149495480320, + "reserved_bytes_peak": 149495480320, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-01T12:58:41.857803Z", + "world_size": 8 + }, + "allocated_bytes_current": 61319299072, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 115704547328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72526018560, + "active_bytes_peak": 133855589376, + "allocated_bytes_current": 72526018560, + "allocated_bytes_peak": 115704547328, + "device": 4, + "inactive_split_bytes_current": 18744133632, + "inactive_split_bytes_peak": 21257594880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 148602093568, + "reserved_bytes_peak": 148602093568, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T12:57:42.616488Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9813999616, + "inactive_split_bytes_current_delta": -8930134016, + "inactive_split_bytes_peak": 21257594880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 149495480320, + "reserved_bytes_current_delta": 893386752, + "reserved_bytes_peak": 149495480320, + "reserved_bytes_peak_delta": 893386752, + "segment_count_current": 551, + "segment_count_current_delta": 6, + "segment_count_peak": 551, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T125841858294Z.pickle", + "timestamp_utc": "2026-08-01T12:58:41.945455Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T125841858294Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T125841858294Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..666f6b7ce412d6bb3d0e43fe8c09ff2ae3656965 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T125841858294Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af9fbd421bdfa0cfe0683e2d19c399db14f0dc560d7cd411d9c70ec8f5ac03d1 +size 3949925 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T125731533519Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T125731533519Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a3ced18e0e749b14c2d5901236bb72c2453af65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T125731533519Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 92058284032, + "active_bytes_peak_delta": 56591947776, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92058284032, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92058284032, + "device": 5, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_peak": 13390509568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 95282003968, + "reserved_bytes_peak": 95282003968, + "segment_count_current": 371, + "segment_count_peak": 371, + "timestamp_utc": "2026-08-01T12:57:31.532915Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 92058284032, + "allocated_bytes_peak_delta": 56591947776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:56:23.360494Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_current_delta": 6011149312, + "inactive_split_bytes_peak": 13390509568, + "inactive_split_bytes_peak_delta": 13230123520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 95282003968, + "reserved_bytes_current_delta": 56037998592, + "reserved_bytes_peak": 95282003968, + "reserved_bytes_peak_delta": 56037998592, + "segment_count_current": 371, + "segment_count_current_delta": 300, + "segment_count_peak": 371, + "segment_count_peak_delta": 300, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T125731533519Z.pickle", + "timestamp_utc": "2026-08-01T12:57:31.652140Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T125731533519Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T125731533519Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..46eff73cbc6ed5b1a7263f54e7ca00926ed519b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T125731533519Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:681c7161b2893e959c41bfb3f014d44da03185dac96e18f5d180f856a022bd61 +size 5766536 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T125732832957Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T125732832957Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e0434b111a54ab1a1c3daedd9c35dedec8438fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T125732832957Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115695290368, + "active_bytes_peak_delta": 23637006336, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115695290368, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115695290368, + "device": 5, + "inactive_split_bytes_current": 7020577792, + "inactive_split_bytes_peak": 15491064832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:32.832560Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115695290368, + "allocated_bytes_peak_delta": 23637006336, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92058284032, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92058284032, + "device": 5, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_peak": 13390509568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 95282003968, + "reserved_bytes_peak": 95282003968, + "segment_count_current": 371, + "segment_count_peak": 371, + "timestamp_utc": "2026-08-01T12:57:31.672781Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7020577792, + "inactive_split_bytes_current_delta": 876609536, + "inactive_split_bytes_peak": 15491064832, + "inactive_split_bytes_peak_delta": 2100555264, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118669443072, + "reserved_bytes_current_delta": 23387439104, + "reserved_bytes_peak": 118669443072, + "reserved_bytes_peak_delta": 23387439104, + "segment_count_current": 425, + "segment_count_current_delta": 54, + "segment_count_peak": 425, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T125732832957Z.pickle", + "timestamp_utc": "2026-08-01T12:57:32.943900Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T125732832957Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T125732832957Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ac4cabd901e2e7eab8c6d9bc4c8005e1a8491e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T125732832957Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39b1d133258c0208beebd216f8c94db0a67eda3cf1f2be0f2d12935361e3206a +size 5891018 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T125734132999Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T125734132999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0bd84fc0d2ffa055c7e47170a9a94e38ed55f02b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T125734132999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115705634816, + "active_bytes_peak_delta": 10344448, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115705634816, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 8492778496, + "inactive_split_bytes_peak": 15550785536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:34.132592Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 10344448, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115695290368, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115695290368, + "device": 5, + "inactive_split_bytes_current": 7020577792, + "inactive_split_bytes_peak": 15491064832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:32.959854Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8492778496, + "inactive_split_bytes_current_delta": 1472200704, + "inactive_split_bytes_peak": 15550785536, + "inactive_split_bytes_peak_delta": 59720704, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118669443072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118669443072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 425, + "segment_count_current_delta": 0, + "segment_count_peak": 425, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T125734132999Z.pickle", + "timestamp_utc": "2026-08-01T12:57:34.242943Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T125734132999Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T125734132999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..277c88afb786881fd7164ce70b9ad6465dc00d28 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T125734132999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71ebfd761a90031f25df06a68420da3259c75ffe5223730e08ef6faba839a8c4 +size 5890181 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T125737022197Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T125737022197Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0245d3e686cdb3c4c1f38db04e545e22ad20cb69 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T125737022197Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 18153318400, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:37.021678Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115705634816, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 8492778496, + "inactive_split_bytes_peak": 15550785536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:34.262396Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 10661654528, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 4989324288, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 31784435712, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 31784435712, + "segment_count_current": 547, + "segment_count_current_delta": 122, + "segment_count_peak": 547, + "segment_count_peak_delta": 122, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T125737022197Z.pickle", + "timestamp_utc": "2026-08-01T12:57:37.152569Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T125737022197Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T125737022197Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f453f60a0596d1da592fdc81265623f3ce8fe1bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T125737022197Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3d72ac734c23c93776162362a52c37e998dd96cb6845ae483357286f3f312e5 +size 6153166 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T125738257444Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T125738257444Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3239840fee5a93140965501bcb12b8821974c538 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T125738257444Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:38.256631Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:37.173456Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T125738257444Z.pickle", + "timestamp_utc": "2026-08-01T12:57:38.261732Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T125738257444Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T125738257444Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a50dbfcfa02372535953698b87c6048bebd8693d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T125738257444Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca8c8786d2dbc94bee8cf039c7abb8bbc47d6b2cf08a9698c1583e3c3b7ad383 +size 146173 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T125739347396Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T125739347396Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6eb06185f1d6a35067196d702c3bf559fecccefe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T125739347396Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:39.346963Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:38.262692Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T125739347396Z.pickle", + "timestamp_utc": "2026-08-01T12:57:39.351680Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T125739347396Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T125739347396Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..757b1f40f1e7cdf752738310dfed8f7380960ab7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T125739347396Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab8db6a34d75e88096ed6ac28fe636d5dc6d0ba1f815bfc3e51e206fa88c41e4 +size 146204 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T125740440513Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T125740440513Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eb35fa1d5bb90c1646e67db94f85ba81140ba7ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T125740440513Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:40.440101Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:39.352550Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T125740440513Z.pickle", + "timestamp_utc": "2026-08-01T12:57:40.444809Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T125740440513Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T125740440513Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f06e037093ceed0d0fcf1e9c1870d1adb8e71bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T125740440513Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:072c1ceeea3be0fee2bec41b682a7cf02885d65223e2522b3502c8256c0c60fd +size 146173 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T125741524282Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T125741524282Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a8a0f6451a57dc54736cc3699f3ec1ef1a088dde --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T125741524282Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:41.523880Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:40.445671Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T125741524282Z.pickle", + "timestamp_utc": "2026-08-01T12:57:41.528512Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T125741524282Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T125741524282Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9091e092eb89f49536cb0b4ac853434fc949fa6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T125741524282Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fbe3ab1dfeb687baf4b97b9266c8fca434dd4d6e4237855e5564666e7f8443c +size 146204 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T125742611361Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T125742611361Z.json new file mode 100644 index 0000000000000000000000000000000000000000..64496b54543d855b0be99458e04aa9fcad30fd9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T125742611361Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:42.610826Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:41.529355Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T125742611361Z.pickle", + "timestamp_utc": "2026-08-01T12:57:42.616391Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T125742611361Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T125742611361Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e62453e737107d665c8e28bec6598433d49d36df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T125742611361Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65dafb0e537ab191d8ce23752bbc789a212487b28ae8ee6f1c0a66689a82566d +size 146173 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T125841858293Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T125841858293Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a2e8694b4116bf1ba0d9a31bf210e6bf9ee07200 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T125841858293Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61317944320, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61317944320, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 61317944320, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 11281263616, + "inactive_split_bytes_peak": 20540109824, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 150523084800, + "reserved_bytes_peak": 150523084800, + "segment_count_current": 552, + "segment_count_peak": 552, + "timestamp_utc": "2026-08-01T12:58:41.857799Z", + "world_size": 8 + }, + "allocated_bytes_current": 61317944320, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 5, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:42.617289Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11281263616, + "inactive_split_bytes_current_delta": -7873169408, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 150523084800, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 150523084800, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 552, + "segment_count_current_delta": 5, + "segment_count_peak": 552, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T125841858293Z.pickle", + "timestamp_utc": "2026-08-01T12:58:41.945917Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T125841858293Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T125841858293Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9bf32555b9a9fd99d1da15bf1b48105206d9e104 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T125841858293Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f7d319cedb85b06c9c894480e758ca47ba3effbda0c4ea7df31350b58918f16 +size 3941122 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T125731533518Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T125731533518Z.json new file mode 100644 index 0000000000000000000000000000000000000000..29c09c41b3e7b88cb2068053aff0c7a6ae5313f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T125731533518Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 92058284032, + "active_bytes_peak_delta": 56591947776, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92058284032, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92058284032, + "device": 6, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_peak": 13390509568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 95282003968, + "reserved_bytes_peak": 95282003968, + "segment_count_current": 371, + "segment_count_peak": 371, + "timestamp_utc": "2026-08-01T12:57:31.532907Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 92058284032, + "allocated_bytes_peak_delta": 56591947776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:56:23.361259Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_current_delta": 6011149312, + "inactive_split_bytes_peak": 13390509568, + "inactive_split_bytes_peak_delta": 13230123520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 95282003968, + "reserved_bytes_current_delta": 56037998592, + "reserved_bytes_peak": 95282003968, + "reserved_bytes_peak_delta": 56037998592, + "segment_count_current": 371, + "segment_count_current_delta": 300, + "segment_count_peak": 371, + "segment_count_peak_delta": 300, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T125731533518Z.pickle", + "timestamp_utc": "2026-08-01T12:57:31.653256Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T125731533518Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T125731533518Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c90d08b85b2159287978d7c502f924caaa879d97 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T125731533518Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28e4f61239c5ba6075e0c8c9bf5743e718e355d68c50f2f98e7c5136629b03d0 +size 5766530 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T125732832987Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T125732832987Z.json new file mode 100644 index 0000000000000000000000000000000000000000..35fc3169b54f410c2b19e99b9285c190f625bd65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T125732832987Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115695290368, + "active_bytes_peak_delta": 23637006336, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115695290368, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115695290368, + "device": 6, + "inactive_split_bytes_current": 7020577792, + "inactive_split_bytes_peak": 15491064832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:32.832569Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115695290368, + "allocated_bytes_peak_delta": 23637006336, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92058284032, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92058284032, + "device": 6, + "inactive_split_bytes_current": 6143968256, + "inactive_split_bytes_peak": 13390509568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 95282003968, + "reserved_bytes_peak": 95282003968, + "segment_count_current": 371, + "segment_count_peak": 371, + "timestamp_utc": "2026-08-01T12:57:31.674117Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7020577792, + "inactive_split_bytes_current_delta": 876609536, + "inactive_split_bytes_peak": 15491064832, + "inactive_split_bytes_peak_delta": 2100555264, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118669443072, + "reserved_bytes_current_delta": 23387439104, + "reserved_bytes_peak": 118669443072, + "reserved_bytes_peak_delta": 23387439104, + "segment_count_current": 425, + "segment_count_current_delta": 54, + "segment_count_peak": 425, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T125732832987Z.pickle", + "timestamp_utc": "2026-08-01T12:57:32.942464Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T125732832987Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T125732832987Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ee04bbc1da805d9b4050f1fc0cf41d0fa5a6cf8d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T125732832987Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce191a50ec82aa9f11d6c1e866d240629d9f12cd49bc3f07d5cd94dbae43d460 +size 5891020 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T125734132994Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T125734132994Z.json new file mode 100644 index 0000000000000000000000000000000000000000..74b1f85a4a9f066332240be7787e50f3ac33daa4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T125734132994Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115705634816, + "active_bytes_peak_delta": 10344448, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115705634816, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 8492778496, + "inactive_split_bytes_peak": 15550785536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:34.132579Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 10344448, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115695290368, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115695290368, + "device": 6, + "inactive_split_bytes_current": 7020577792, + "inactive_split_bytes_peak": 15491064832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:32.958614Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8492778496, + "inactive_split_bytes_current_delta": 1472200704, + "inactive_split_bytes_peak": 15550785536, + "inactive_split_bytes_peak_delta": 59720704, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118669443072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118669443072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 425, + "segment_count_current_delta": 0, + "segment_count_peak": 425, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T125734132994Z.pickle", + "timestamp_utc": "2026-08-01T12:57:34.241914Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T125734132994Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T125734132994Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..52b09f8c4d5e82e20a586fdf84af3eb7d9498a91 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T125734132994Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3609982feb63fb1558315e0ebf7b96f0fd4eb2f4d57ac1c59a10953994a57036 +size 5890179 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T125737022090Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T125737022090Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27eb76d1824bd92d41b0cd1dcbee7de1fb2e8b93 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T125737022090Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 18153318400, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:37.021548Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 115705634816, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 8492778496, + "inactive_split_bytes_peak": 15550785536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:34.257682Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 10661654528, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 4989324288, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 31784435712, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 31784435712, + "segment_count_current": 547, + "segment_count_current_delta": 122, + "segment_count_peak": 547, + "segment_count_peak_delta": 122, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T125737022090Z.pickle", + "timestamp_utc": "2026-08-01T12:57:37.145388Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T125737022090Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T125737022090Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6db816a70fa8a56cb35dc5cbb4d3d13bb78faacc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T125737022090Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:936ce394478d46d512360c1d05237114440c31b90f19da1f6d2c8bab1c167db0 +size 6153165 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T125738257408Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T125738257408Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9802fb7bf637f6b4b3dd8d1632e4096c656c3923 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T125738257408Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:38.256588Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:37.165081Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T125738257408Z.pickle", + "timestamp_utc": "2026-08-01T12:57:38.261686Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T125738257408Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T125738257408Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..53d741ac4b7f9ddae2f3f42b7d036f96e63cbd5d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T125738257408Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a61fc263aac2ecdbcae11b727d6514e53708d9c1104eae7ec659f9b0ca676e9a +size 146172 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T125739347418Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T125739347418Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ce66c40a91e853dfee092801c3d1f9c0225aa2fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T125739347418Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:39.346985Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:38.262560Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T125739347418Z.pickle", + "timestamp_utc": "2026-08-01T12:57:39.351634Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T125739347418Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T125739347418Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b3aa8a1056d297e3383960f3ec33149e7537738 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T125739347418Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1ef115718276b58c5835913b59202f68e9525bd9eb0f79892cc861ca755b1bc +size 146203 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T125740440604Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T125740440604Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d3a1a9202ad51fb9018f5fb52a6898d9499e6dce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T125740440604Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:40.440122Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:39.352535Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T125740440604Z.pickle", + "timestamp_utc": "2026-08-01T12:57:40.445488Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T125740440604Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T125740440604Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c26858b120ed1a34ff06510fd5c0dd954fbe056a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T125740440604Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1af21d3a6dc09dc9b58a6018b229320b8f59cf6f358ded12a70692fbd16652aa +size 146172 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T125741524301Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T125741524301Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c1e8be584a6266cc33c4ebedcf7325052d0f81d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T125741524301Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:41.523880Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:40.446393Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T125741524301Z.pickle", + "timestamp_utc": "2026-08-01T12:57:41.528536Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T125741524301Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T125741524301Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e60da318556ad5d16dfa041eb5477ec9c81c7f92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T125741524301Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c52638c1bf6fd49d51798450530f73c5776a3d83148555e0674f5d70a783cd7e +size 146203 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T125742611160Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T125742611160Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ada14474c3ea4be1a9a63d21261186c2aea24806 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T125742611160Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524663808, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:42.610718Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524663808, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:41.529419Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T125742611160Z.pickle", + "timestamp_utc": "2026-08-01T12:57:42.615672Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T125742611160Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T125742611160Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e49808fe2d79f8b01e338c96fdbbe818ca26fe0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T125742611160Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b51a88773bb2ddbcab2c04f0a2cd1c4402c93ed47e5c06a4545bb258340b5a2 +size 146172 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T125841858378Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T125841858378Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a114b721cb6fbc2ab3b7dfac167233d102a44903 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T125841858378Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61317944320, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 133858953216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61317944320, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 61317944320, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 11281263616, + "inactive_split_bytes_peak": 20540109824, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 150523084800, + "reserved_bytes_peak": 150523084800, + "segment_count_current": 552, + "segment_count_peak": 552, + "timestamp_utc": "2026-08-01T12:58:41.857812Z", + "world_size": 8 + }, + "allocated_bytes_current": 61317944320, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 115705634816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524663808, + "active_bytes_peak": 133858953216, + "allocated_bytes_current": 72524663808, + "allocated_bytes_peak": 115705634816, + "device": 6, + "inactive_split_bytes_current": 19154433024, + "inactive_split_bytes_peak": 20540109824, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:42.616602Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 11281263616, + "inactive_split_bytes_current_delta": -7873169408, + "inactive_split_bytes_peak": 20540109824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 150523084800, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 150523084800, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 552, + "segment_count_current_delta": 5, + "segment_count_peak": 552, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T125841858378Z.pickle", + "timestamp_utc": "2026-08-01T12:58:41.944097Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T125841858378Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T125841858378Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..733e303057a3e491a79ad204f0c002defbbeaa47 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T125841858378Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:674ac04d681a2ea02e0f16f357f0577f3e5f093782535ff098f41d165fd60d90 +size 3941123 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T125731533543Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T125731533543Z.json new file mode 100644 index 0000000000000000000000000000000000000000..19d1a8d8c892bb7eedb4f440de4a2b1c5e325cae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T125731533543Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 37276165632, + "active_bytes_peak": 92058284032, + "active_bytes_peak_delta": 56591947776, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 92058284032, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 92058284032, + "device": 7, + "inactive_split_bytes_current": 6143967744, + "inactive_split_bytes_peak": 13390509568, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 95282003968, + "reserved_bytes_peak": 95282003968, + "segment_count_current": 371, + "segment_count_peak": 371, + "timestamp_utc": "2026-08-01T12:57:31.532960Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 37276165632, + "allocated_bytes_peak": 92058284032, + "allocated_bytes_peak_delta": 56591947776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T12:56:23.361170Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6143967744, + "inactive_split_bytes_current_delta": 6011148800, + "inactive_split_bytes_peak": 13390509568, + "inactive_split_bytes_peak_delta": 13230123520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 95282003968, + "reserved_bytes_current_delta": 56037998592, + "reserved_bytes_peak": 95282003968, + "reserved_bytes_peak_delta": 56037998592, + "segment_count_current": 371, + "segment_count_current_delta": 300, + "segment_count_peak": 371, + "segment_count_peak_delta": 300, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T125731533543Z.pickle", + "timestamp_utc": "2026-08-01T12:57:31.649673Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T125731533543Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T125731533543Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..feb7eadb0bb2f5b6342372bdf32ae02fa0473197 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T125731533543Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de6f45ec6bfaea79ef32df3e4b7ad21039bbd71fc8d51b292012e405ed4e546f +size 5768132 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T125732833046Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T125732833046Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a5447332dace4273325a621ea51a385aaae95e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T125732833046Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115695290880, + "active_bytes_peak_delta": 23637006848, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 115695290880, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 115695290880, + "device": 7, + "inactive_split_bytes_current": 7020577280, + "inactive_split_bytes_peak": 15493161472, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:32.832639Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115695290880, + "allocated_bytes_peak_delta": 23637006848, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 92058284032, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 92058284032, + "device": 7, + "inactive_split_bytes_current": 6143967744, + "inactive_split_bytes_peak": 13390509568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 95282003968, + "reserved_bytes_peak": 95282003968, + "segment_count_current": 371, + "segment_count_peak": 371, + "timestamp_utc": "2026-08-01T12:57:31.666793Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7020577280, + "inactive_split_bytes_current_delta": 876609536, + "inactive_split_bytes_peak": 15493161472, + "inactive_split_bytes_peak_delta": 2102651904, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118669443072, + "reserved_bytes_current_delta": 23387439104, + "reserved_bytes_peak": 118669443072, + "reserved_bytes_peak_delta": 23387439104, + "segment_count_current": 425, + "segment_count_current_delta": 54, + "segment_count_peak": 425, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T125732833046Z.pickle", + "timestamp_utc": "2026-08-01T12:57:32.940278Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T125732833046Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T125732833046Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..41c2e108fdd7cb1e62afa6149fe176ea35db1b91 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T125732833046Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c73de61e95b05a8841eac6f9ed1474c37a47543066b95ac40237147d5fbf0ef +size 5892734 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T125734133084Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T125734133084Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f1d6c43c35a0912b65b36ef340ea09a0a2166043 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T125734133084Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115705635328, + "active_bytes_peak_delta": 10344448, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 115705635328, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 8492777984, + "inactive_split_bytes_peak": 15548687872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:34.132669Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705635328, + "allocated_bytes_peak_delta": 10344448, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 115695290880, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 115695290880, + "device": 7, + "inactive_split_bytes_current": 7020577280, + "inactive_split_bytes_peak": 15493161472, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:32.954286Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8492777984, + "inactive_split_bytes_current_delta": 1472200704, + "inactive_split_bytes_peak": 15548687872, + "inactive_split_bytes_peak_delta": 55526400, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118669443072, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118669443072, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 425, + "segment_count_current_delta": 0, + "segment_count_peak": 425, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T125734133084Z.pickle", + "timestamp_utc": "2026-08-01T12:57:34.241985Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T125734133084Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T125734133084Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..16d2c3ba68d34b806973a1d7996ccc45da4c7799 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T125734133084Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1970ca04c3448a6dd5c2149a535864497f198abc98d39ef31c2e967913a93fa9 +size 5891961 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T125737022147Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T125737022147Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6126b42322767ba1eeb471711771a4fde0278e77 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T125737022147Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524664320, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 133858953728, + "active_bytes_peak_delta": 18153318400, + "after": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_peak": 20540109312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:37.021621Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524664320, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115705635328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 115705635328, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 8492777984, + "inactive_split_bytes_peak": 15548687872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118669443072, + "reserved_bytes_peak": 118669443072, + "segment_count_current": 425, + "segment_count_peak": 425, + "timestamp_utc": "2026-08-01T12:57:34.255762Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_current_delta": 10661654528, + "inactive_split_bytes_peak": 20540109312, + "inactive_split_bytes_peak_delta": 4991421440, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 31784435712, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 31784435712, + "segment_count_current": 547, + "segment_count_current_delta": 122, + "segment_count_peak": 547, + "segment_count_peak_delta": 122, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T125737022147Z.pickle", + "timestamp_utc": "2026-08-01T12:57:37.142927Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T125737022147Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T125737022147Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a68ad0398c3607cdc35f50d8ce73a306aed67751 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T125737022147Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a7855de3478d53fabd0f7fa8a6a94a44c65e7bacd70dc2a6ceab520587cfae4 +size 6154911 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T125738257722Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T125738257722Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d795db4ad9428b39ee310ae75c687b584997d1c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T125738257722Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524664320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_peak": 20540109312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:38.256826Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524664320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705635328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_peak": 20540109312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:37.157928Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T125738257722Z.pickle", + "timestamp_utc": "2026-08-01T12:57:38.263258Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T125738257722Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T125738257722Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27121423ee356ecf59d6e698b39cd60021e3d934 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T125738257722Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95a3cc8c4936b0b8302d084c4a1677b75d216c57927056623d0d544033642b07 +size 147918 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T125739347497Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T125739347497Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6bc2921401fd60da4c4d390752a9de3e515a5a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T125739347497Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524664320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_peak": 20540109312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:39.347068Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524664320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705635328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_peak": 20540109312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:38.264177Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T125739347497Z.pickle", + "timestamp_utc": "2026-08-01T12:57:39.351828Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T125739347497Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T125739347497Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4023a0f9d62a9c58bdef2564a4514fd70ce32c2a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T125739347497Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cadb03e9c57d0df33cb6a4aed44960c219f17bf95df5054928e61a96b5fd3e41 +size 147949 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T125740440689Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T125740440689Z.json new file mode 100644 index 0000000000000000000000000000000000000000..864f536d8a19ce6b72098f7ca0545843cae50650 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T125740440689Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524664320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_peak": 20540109312, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:40.440227Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524664320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705635328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_peak": 20540109312, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:39.352704Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T125740440689Z.pickle", + "timestamp_utc": "2026-08-01T12:57:40.445440Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T125740440689Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T125740440689Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2901ab6dfaaa955c9201c810ef85700c0e93918 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T125740440689Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f1f838b37693db529b461138c94f94f6845db361ff3152bf33e346cf5a61eb9 +size 147949 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T125741524370Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T125741524370Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8dc981ae4cfd4c0f8cadefb6ea63d1a8026b586a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T125741524370Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524664320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_peak": 20540109312, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:41.523944Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524664320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705635328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_peak": 20540109312, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:40.446337Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T125741524370Z.pickle", + "timestamp_utc": "2026-08-01T12:57:41.528688Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T125741524370Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T125741524370Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..040ce8bf168832921474514224d5c4f0b6f6fa3f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T125741524370Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eac99cfe185cf14da5b19ab944dde1163628480f0179e7ec338b9d3d75929b92 +size 147918 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T125742611275Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T125742611275Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ff21590cd7e42267788d0004add821ecc3e32719 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T125742611275Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72524664320, + "active_bytes_current_delta": 0, + "active_bytes_peak": 133858953728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_peak": 20540109312, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:42.610840Z", + "world_size": 8 + }, + "allocated_bytes_current": 72524664320, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705635328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_peak": 20540109312, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:41.529539Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 20540109312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 150453878784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 150453878784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 547, + "segment_count_current_delta": 0, + "segment_count_peak": 547, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T125742611275Z.pickle", + "timestamp_utc": "2026-08-01T12:57:42.615731Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T125742611275Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T125742611275Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..06f701a0d76b1f1569a24bc3ff197b46d5e438b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T125742611275Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1a27e23b07a991eba9a4e9f3dd703f323f123e18ba75b650e51fd1632fc12d1 +size 147949 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T125841858276Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T125841858276Z.json new file mode 100644 index 0000000000000000000000000000000000000000..af9dff6cec8b6354eb7a94f29d2bb69be871c037 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T125841858276Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61317944832, + "active_bytes_current_delta": -11206719488, + "active_bytes_peak": 133858953728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61317944832, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 61317944832, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 11281263104, + "inactive_split_bytes_peak": 20540109312, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 150523084800, + "reserved_bytes_peak": 150523084800, + "segment_count_current": 552, + "segment_count_peak": 552, + "timestamp_utc": "2026-08-01T12:58:41.857819Z", + "world_size": 8 + }, + "allocated_bytes_current": 61317944832, + "allocated_bytes_current_delta": -11206719488, + "allocated_bytes_peak": 115705635328, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72524664320, + "active_bytes_peak": 133858953728, + "allocated_bytes_current": 72524664320, + "allocated_bytes_peak": 115705635328, + "device": 7, + "inactive_split_bytes_current": 19154432512, + "inactive_split_bytes_peak": 20540109312, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 150453878784, + "reserved_bytes_peak": 150453878784, + "segment_count_current": 547, + "segment_count_peak": 547, + "timestamp_utc": "2026-08-01T12:57:42.616644Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11281263104, + "inactive_split_bytes_current_delta": -7873169408, + "inactive_split_bytes_peak": 20540109312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 150523084800, + "reserved_bytes_current_delta": 69206016, + "reserved_bytes_peak": 150523084800, + "reserved_bytes_peak_delta": 69206016, + "segment_count_current": 552, + "segment_count_current_delta": 5, + "segment_count_peak": 552, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T125841858276Z.pickle", + "timestamp_utc": "2026-08-01T12:58:41.941826Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T125841858276Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T125841858276Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..82150ff4b6f57f6541b19541bac0b614ef15697c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T125841858276Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07801b47fb434044a7784818fc2b92fa5341e94c8fad69da54eb81bb39b3d3de +size 3942869 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T130138175491Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T130138175491Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d62a12316ea7d3107332eb417bb5b710f60b44e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T130138175491Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921626624, + "active_bytes_current_delta": 31690171392, + "active_bytes_peak": 92156583936, + "active_bytes_peak_delta": 56690247680, + "after": { + "active_bytes_current": 66921626624, + "active_bytes_peak": 92156583936, + "allocated_bytes_current": 66921626624, + "allocated_bytes_peak": 92156583936, + "device": 0, + "inactive_split_bytes_current": 5430117376, + "inactive_split_bytes_peak": 10894253056, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98811510784, + "reserved_bytes_peak": 98811510784, + "segment_count_current": 439, + "segment_count_peak": 439, + "timestamp_utc": "2026-08-01T13:01:38.174915Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921626624, + "allocated_bytes_current_delta": 31690171392, + "allocated_bytes_peak": 92156583936, + "allocated_bytes_peak_delta": 56690247680, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:00:29.416600Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5430117376, + "inactive_split_bytes_current_delta": 5295201280, + "inactive_split_bytes_peak": 10894253056, + "inactive_split_bytes_peak_delta": 10733621248, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98811510784, + "reserved_bytes_current_delta": 59565408256, + "reserved_bytes_peak": 98811510784, + "reserved_bytes_peak_delta": 59565408256, + "segment_count_current": 439, + "segment_count_current_delta": 367, + "segment_count_peak": 439, + "segment_count_peak_delta": 367, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T130138175491Z.pickle", + "timestamp_utc": "2026-08-01T13:01:38.287823Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T130138175491Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T130138175491Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0964d23b5da4c3d20cc57212cb9e9be910ad3421 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T130138175491Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:993c684b707ff58ca91c8e0f2e2b9d9f8468040a39f8d7bc0257d4370493357c +size 5395136 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T130139338842Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T130139338842Z.json new file mode 100644 index 0000000000000000000000000000000000000000..43e8b4ceb007dee4fc9369eb12488e0d99cdc443 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T130139338842Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921626624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115789879296, + "active_bytes_peak_delta": 23633295360, + "after": { + "active_bytes_current": 66921626624, + "active_bytes_peak": 115789879296, + "allocated_bytes_current": 66921626624, + "allocated_bytes_peak": 115789879296, + "device": 0, + "inactive_split_bytes_current": 4979229696, + "inactive_split_bytes_peak": 10894253056, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119430709248, + "reserved_bytes_peak": 119430709248, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:01:39.338433Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921626624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115789879296, + "allocated_bytes_peak_delta": 23633295360, + "before": { + "active_bytes_current": 66921626624, + "active_bytes_peak": 92156583936, + "allocated_bytes_current": 66921626624, + "allocated_bytes_peak": 92156583936, + "device": 0, + "inactive_split_bytes_current": 5430117376, + "inactive_split_bytes_peak": 10894253056, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98811510784, + "reserved_bytes_peak": 98811510784, + "segment_count_current": 439, + "segment_count_peak": 439, + "timestamp_utc": "2026-08-01T13:01:38.307089Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4979229696, + "inactive_split_bytes_current_delta": -450887680, + "inactive_split_bytes_peak": 10894253056, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119430709248, + "reserved_bytes_current_delta": 20619198464, + "reserved_bytes_peak": 119430709248, + "reserved_bytes_peak_delta": 20619198464, + "segment_count_current": 496, + "segment_count_current_delta": 57, + "segment_count_peak": 496, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T130139338842Z.pickle", + "timestamp_utc": "2026-08-01T13:01:39.446460Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T130139338842Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T130139338842Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ce4aa746fa336f30c7eafbb8a00e5e3197640af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T130139338842Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4e5b86f824d111456790a03b0c0a258b50b81050f2d1074b860b5d73a252183 +size 5587002 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T130140501423Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T130140501423Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c7dbb121afdd683d71178da38e03ffecf2948fb4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T130140501423Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921626624, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115800848384, + "active_bytes_peak_delta": 10969088, + "after": { + "active_bytes_current": 66921626624, + "active_bytes_peak": 115800848384, + "allocated_bytes_current": 66921626624, + "allocated_bytes_peak": 115800848384, + "device": 0, + "inactive_split_bytes_current": 5209916416, + "inactive_split_bytes_peak": 10894253056, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 120135352320, + "reserved_bytes_peak": 120135352320, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T13:01:40.501020Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921626624, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115800848384, + "allocated_bytes_peak_delta": 10969088, + "before": { + "active_bytes_current": 66921626624, + "active_bytes_peak": 115789879296, + "allocated_bytes_current": 66921626624, + "allocated_bytes_peak": 115789879296, + "device": 0, + "inactive_split_bytes_current": 4979229696, + "inactive_split_bytes_peak": 10894253056, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 119430709248, + "reserved_bytes_peak": 119430709248, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:01:39.461603Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5209916416, + "inactive_split_bytes_current_delta": 230686720, + "inactive_split_bytes_peak": 10894253056, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 120135352320, + "reserved_bytes_current_delta": 704643072, + "reserved_bytes_peak": 120135352320, + "reserved_bytes_peak_delta": 704643072, + "segment_count_current": 499, + "segment_count_current_delta": 3, + "segment_count_peak": 499, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T130140501423Z.pickle", + "timestamp_utc": "2026-08-01T13:01:40.605437Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T130140501423Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T130140501423Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ecd2986db97c4b4c65a160ed5e3f44e40dfc15a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T130140501423Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89b887643aec403bbe324dad9f0faa7a6a5f9665397365fcbc9f5edf13250868 +size 5582163 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T130143261252Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T130143261252Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1138ac347f23173f68d93e0f751bd4d983d3b273 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T130143261252Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938670080, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 167776659456, + "active_bytes_peak_delta": 51975811072, + "after": { + "active_bytes_current": 66938670080, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66938670080, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_peak": 14429191680, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:43.260721Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938670080, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115805144576, + "allocated_bytes_peak_delta": 4296192, + "before": { + "active_bytes_current": 66921626624, + "active_bytes_peak": 115800848384, + "allocated_bytes_current": 66921626624, + "allocated_bytes_peak": 115800848384, + "device": 0, + "inactive_split_bytes_current": 5209916416, + "inactive_split_bytes_peak": 10894253056, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 120135352320, + "reserved_bytes_peak": 120135352320, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T13:01:40.620098Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_current_delta": 22802432, + "inactive_split_bytes_peak": 14429191680, + "inactive_split_bytes_peak_delta": 3534938624, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 61117300736, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 61117300736, + "segment_count_current": 802, + "segment_count_current_delta": 303, + "segment_count_peak": 802, + "segment_count_peak_delta": 303, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T130143261252Z.pickle", + "timestamp_utc": "2026-08-01T13:01:43.380750Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T130143261252Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T130143261252Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e168e5041b03dcfc5bb96777444d4bd7c4b1b094 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T130143261252Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2199b1d7fb8ef93ee9d20928132c840f35adbc19a5913bc228806322df9cfaf5 +size 5848776 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T130144227778Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T130144227778Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15e1e100a5e39cc4ab86bf0816ad7c2d2ddb978d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T130144227778Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938670080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167776659456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938670080, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66938670080, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_peak": 14429191680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:44.227327Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938670080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115805144576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938670080, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66938670080, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_peak": 14429191680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:43.397292Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14429191680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T130144227778Z.pickle", + "timestamp_utc": "2026-08-01T13:01:44.234425Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T130144227778Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T130144227778Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0d1d6b62a7e88cdacf6517bef391011b7763653 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T130144227778Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eab7269b86cfd6d75962768bc3575427b87bdf5590ea3cd673cff7155c3f9d53 +size 214710 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T130145065524Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T130145065524Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e19e69402e5f44714c38ec3925e95d45a7044664 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T130145065524Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938670080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167776659456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938670080, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66938670080, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_peak": 14429191680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:45.065090Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938670080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115805144576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938670080, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66938670080, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_peak": 14429191680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:44.235398Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14429191680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T130145065524Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.071827Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T130145065524Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T130145065524Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6a6fd0c0f11ac7fa8c8cf8174b2bbff38afe431 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T130145065524Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d87af3af02ac7ff04ca21f95c82cc718c3530e8d5a1162edf32d95f9cff088e +size 214679 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T130145901589Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T130145901589Z.json new file mode 100644 index 0000000000000000000000000000000000000000..98308225027f9d29a470ba81ca1e642158e2ed9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T130145901589Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938670080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167776659456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938670080, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66938670080, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_peak": 14429191680, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:45.901189Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938670080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115805144576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938670080, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66938670080, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_peak": 14429191680, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:45.072769Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14429191680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T130145901589Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.907497Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T130145901589Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T130145901589Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de460e944ddd53721eb6c574c9a570bad65ffb47 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T130145901589Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6e880adefc89e6d3237a877b5fee0cf2a0e670b70bb8aac8474c6230c29bb7f +size 214710 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T130146731469Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T130146731469Z.json new file mode 100644 index 0000000000000000000000000000000000000000..efef1b572cd07ec0b2738cf52bc87ec401e7ac60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T130146731469Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938670080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167776659456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938670080, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66938670080, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_peak": 14429191680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:46.731066Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938670080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115805144576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938670080, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66938670080, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_peak": 14429191680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:45.908437Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14429191680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T130146731469Z.pickle", + "timestamp_utc": "2026-08-01T13:01:46.737310Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T130146731469Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T130146731469Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf7d94aff7ff73c1bb2ccda07110d473b28e3546 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T130146731469Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c756127753288a8a75297ad7991b2d3a33eff568416ae62af62f81f087a56d90 +size 214679 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T130147563599Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T130147563599Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c4780e1e039015e0e63389479137c9982cd55cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T130147563599Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938670080, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167776659456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938670080, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66938670080, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_peak": 14429191680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:47.563214Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938670080, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115805144576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938670080, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66938670080, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_peak": 14429191680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:46.738260Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14429191680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T130147563599Z.pickle", + "timestamp_utc": "2026-08-01T13:01:47.569510Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T130147563599Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T130147563599Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b0241926da91fcafc516db0c795e6e730dd8e8f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T130147563599Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a29810c639a2ac75d15fb550bf75b8185037166c05dd3fffc169a74b85035e17 +size 214710 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T130149798344Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T130149798344Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7415235ec502cef5c1f84165d1fb74d2718b5282 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T130149798344Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955762688, + "active_bytes_current_delta": 17092608, + "active_bytes_peak": 167776659456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955762688, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66955762688, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 4970259456, + "inactive_split_bytes_peak": 14429191680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 4, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 182762602496, + "reserved_bytes_peak": 182762602496, + "segment_count_current": 767, + "segment_count_peak": 808, + "timestamp_utc": "2026-08-01T13:01:49.797928Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955762688, + "allocated_bytes_current_delta": 17092608, + "allocated_bytes_peak": 115805144576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938670080, + "active_bytes_peak": 167776659456, + "allocated_bytes_current": 66938670080, + "allocated_bytes_peak": 115805144576, + "device": 0, + "inactive_split_bytes_current": 5232718848, + "inactive_split_bytes_peak": 14429191680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:47.570458Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4970259456, + "inactive_split_bytes_current_delta": -262459392, + "inactive_split_bytes_peak": 14429191680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 4, + "num_alloc_retries_delta": 4, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 182762602496, + "reserved_bytes_current_delta": 1509949440, + "reserved_bytes_peak": 182762602496, + "reserved_bytes_peak_delta": 1509949440, + "segment_count_current": 767, + "segment_count_current_delta": -35, + "segment_count_peak": 808, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T130149798344Z.pickle", + "timestamp_utc": "2026-08-01T13:01:49.872298Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T130149798344Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T130149798344Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..36190685fde3c120aa7ec6551ec1a320c68ace56 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T130149798344Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3401f2dae73d2325e6888731f78d4feeb534b45e3060253e99e1ab378eb74247 +size 4029859 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T130138175680Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T130138175680Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e022d91e64ac135f4ba1c584eb0e1feaa14861ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T130138175680Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 92159424512, + "active_bytes_peak_delta": 56693088256, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92159424512, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92159424512, + "device": 1, + "inactive_split_bytes_current": 5489345536, + "inactive_split_bytes_peak": 10767688704, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 98574532608, + "reserved_bytes_peak": 98574532608, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T13:01:38.175044Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 92159424512, + "allocated_bytes_peak_delta": 56693088256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:00:29.416732Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5489345536, + "inactive_split_bytes_current_delta": 5356526592, + "inactive_split_bytes_peak": 10767688704, + "inactive_split_bytes_peak_delta": 10607302656, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 98574532608, + "reserved_bytes_current_delta": 59330527232, + "reserved_bytes_peak": 98574532608, + "reserved_bytes_peak_delta": 59330527232, + "segment_count_current": 437, + "segment_count_current_delta": 366, + "segment_count_peak": 437, + "segment_count_peak_delta": 366, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T130138175680Z.pickle", + "timestamp_utc": "2026-08-01T13:01:38.291335Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T130138175680Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T130138175680Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..85fffe0beed428c8345786d85e9a918296ae633f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T130138175680Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee533c53c1327308d55385ff144d0a6064e5b7d25f98208a92cf9232904fd7f4 +size 5384736 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T130139339013Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T130139339013Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80770e8ca44fbf203b6e449cfa0baa23307b1b23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T130139339013Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115792877568, + "active_bytes_peak_delta": 23633453056, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115792877568, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115792877568, + "device": 1, + "inactive_split_bytes_current": 5042652160, + "inactive_split_bytes_peak": 10767688704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 119069999104, + "reserved_bytes_peak": 119069999104, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:01:39.338492Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115792877568, + "allocated_bytes_peak_delta": 23633453056, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92159424512, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92159424512, + "device": 1, + "inactive_split_bytes_current": 5489345536, + "inactive_split_bytes_peak": 10767688704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 98574532608, + "reserved_bytes_peak": 98574532608, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T13:01:38.309856Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5042652160, + "inactive_split_bytes_current_delta": -446693376, + "inactive_split_bytes_peak": 10767688704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 119069999104, + "reserved_bytes_current_delta": 20495466496, + "reserved_bytes_peak": 119069999104, + "reserved_bytes_peak_delta": 20495466496, + "segment_count_current": 490, + "segment_count_current_delta": 53, + "segment_count_peak": 490, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T130139339013Z.pickle", + "timestamp_utc": "2026-08-01T13:01:39.445591Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T130139339013Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T130139339013Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..32ac8c675335cdc12b78ad24bbd02b5d97e04d14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T130139339013Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e1edcbece96dd48e6e0509e7883f0f98f9b3377bcbcb542ab60b5215a05fa2b +size 5579617 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T130140501574Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T130140501574Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab926e3d653eb1c17cf29f1da798c022d59c3c5a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T130140501574Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115802054656, + "active_bytes_peak_delta": 9177088, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115802054656, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115802054656, + "device": 1, + "inactive_split_bytes_current": 5273338880, + "inactive_split_bytes_peak": 10767688704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 119304880128, + "reserved_bytes_peak": 119304880128, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:01:40.501108Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115802054656, + "allocated_bytes_peak_delta": 9177088, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115792877568, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115792877568, + "device": 1, + "inactive_split_bytes_current": 5042652160, + "inactive_split_bytes_peak": 10767688704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 119069999104, + "reserved_bytes_peak": 119069999104, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:01:39.459432Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5273338880, + "inactive_split_bytes_current_delta": 230686720, + "inactive_split_bytes_peak": 10767688704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 119304880128, + "reserved_bytes_current_delta": 234881024, + "reserved_bytes_peak": 119304880128, + "reserved_bytes_peak_delta": 234881024, + "segment_count_current": 491, + "segment_count_current_delta": 1, + "segment_count_peak": 491, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T130140501574Z.pickle", + "timestamp_utc": "2026-08-01T13:01:40.608194Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T130140501574Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T130140501574Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e84a1b2793a9004eb112115f16fecafe35ee51d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T130140501574Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2225d46081cb6041597f2cb0acae546f222e3f36112b21b06168201bf36160f7 +size 5578366 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T130143261369Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T130143261369Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ce128becc7053eecb3e0685e59b1a016b60b7f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T130143261369Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 167787161600, + "active_bytes_peak_delta": 51985106944, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167787161600, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115811584000, + "device": 1, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_peak": 13941733376, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:43.260712Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115811584000, + "allocated_bytes_peak_delta": 9529344, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115802054656, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115802054656, + "device": 1, + "inactive_split_bytes_current": 5273338880, + "inactive_split_bytes_peak": 10767688704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 119304880128, + "reserved_bytes_peak": 119304880128, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:01:40.621791Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_current_delta": 16510976, + "inactive_split_bytes_peak": 13941733376, + "inactive_split_bytes_peak_delta": 3174044672, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 181491728384, + "reserved_bytes_current_delta": 62186848256, + "reserved_bytes_peak": 181491728384, + "reserved_bytes_peak_delta": 62186848256, + "segment_count_current": 803, + "segment_count_current_delta": 312, + "segment_count_peak": 803, + "segment_count_peak_delta": 312, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T130143261369Z.pickle", + "timestamp_utc": "2026-08-01T13:01:43.381488Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T130143261369Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T130143261369Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eaa16d6c64e65435826e817ecf363da11d84982f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T130143261369Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc76c6e9bc89a26df8ddad03c3ab71affc1b68c587126fa2cb76bd1dd7d889b5 +size 6574220 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T130144228140Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T130144228140Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae19b20843622463a692b9dae714264262a13bce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T130144228140Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167787161600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167787161600, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115811584000, + "device": 1, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_peak": 13941733376, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:44.227642Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115811584000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167787161600, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115811584000, + "device": 1, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_peak": 13941733376, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:43.396490Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13941733376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 181491728384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181491728384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 803, + "segment_count_current_delta": 0, + "segment_count_peak": 803, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T130144228140Z.pickle", + "timestamp_utc": "2026-08-01T13:01:44.237595Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T130144228140Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T130144228140Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7b52d507dee031130dc9dbdc4d64a752b87ec9d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T130144228140Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05531b275577ebc703354cbe3069fc5964db6b0e2f7bb68fc296747af33ad964 +size 236026 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T130145065482Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T130145065482Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69c3566834c348fbe366a31ad5d4a953d0d0056b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T130145065482Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167787161600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167787161600, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115811584000, + "device": 1, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_peak": 13941733376, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:45.065065Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115811584000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167787161600, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115811584000, + "device": 1, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_peak": 13941733376, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:44.238599Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13941733376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 181491728384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181491728384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 803, + "segment_count_current_delta": 0, + "segment_count_peak": 803, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T130145065482Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.071698Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T130145065482Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T130145065482Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6be77e3db0e0cfa153d1ca1a04a54c1fbf1f0eab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T130145065482Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc3af41085062c0c15620f8c3f5a04be1514171043bf8436b3a4cc39e3d1a165 +size 235995 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T130145901588Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T130145901588Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eb5b7f0772dfad620663fa79714f75ef654a7317 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T130145901588Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167787161600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167787161600, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115811584000, + "device": 1, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_peak": 13941733376, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:45.901188Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115811584000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167787161600, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115811584000, + "device": 1, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_peak": 13941733376, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:45.072683Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13941733376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 181491728384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181491728384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 803, + "segment_count_current_delta": 0, + "segment_count_peak": 803, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T130145901588Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.907408Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T130145901588Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T130145901588Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1078af7a2651b8b6c98e81d14532df14fe7dc970 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T130145901588Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06e2aa0e50b206dc56b47e88b91a2488a1a69b55653e58b8b9270674175b7896 +size 236026 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T130146731480Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T130146731480Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27b54711f95a1af26a9a696b3b2313d01ca90b8f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T130146731480Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167787161600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167787161600, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115811584000, + "device": 1, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_peak": 13941733376, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:46.731047Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115811584000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167787161600, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115811584000, + "device": 1, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_peak": 13941733376, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:45.908378Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13941733376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 181491728384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181491728384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 803, + "segment_count_current_delta": 0, + "segment_count_peak": 803, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T130146731480Z.pickle", + "timestamp_utc": "2026-08-01T13:01:46.737279Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T130146731480Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T130146731480Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e8a3072e58652cb11a665c79af39e8dfd08bee1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T130146731480Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4672b622dfb6280496f58154fa257f03764481e6b21f0d69cc2eb89b66592ad8 +size 235995 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T130147563603Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T130147563603Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ed793c1a7539aa551063601a650ef3d4a8aa875 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T130147563603Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167787161600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167787161600, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115811584000, + "device": 1, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_peak": 13941733376, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:47.563194Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115811584000, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167787161600, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115811584000, + "device": 1, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_peak": 13941733376, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:46.738228Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5289849856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13941733376, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 181491728384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181491728384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 803, + "segment_count_current_delta": 0, + "segment_count_peak": 803, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T130147563603Z.pickle", + "timestamp_utc": "2026-08-01T13:01:47.569859Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T130147563603Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T130147563603Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe0f577749b1549a33b73986648939643a2143f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T130147563603Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15b5915a0b66103f39341cc0a96bede7e9c1981b863c4cd40ecf96a40f34b3f2 +size 236026 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T130138175459Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T130138175459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eab72c9fa1d29a39473c1ff619299a3b7e46d5c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T130138175459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 92153133056, + "active_bytes_peak_delta": 56686796800, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92153133056, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92153133056, + "device": 2, + "inactive_split_bytes_current": 5252367360, + "inactive_split_bytes_peak": 11122107392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98809413632, + "reserved_bytes_peak": 98809413632, + "segment_count_current": 438, + "segment_count_peak": 438, + "timestamp_utc": "2026-08-01T13:01:38.174869Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 92153133056, + "allocated_bytes_peak_delta": 56686796800, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:00:29.416568Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5252367360, + "inactive_split_bytes_current_delta": 5119548416, + "inactive_split_bytes_peak": 11122107392, + "inactive_split_bytes_peak_delta": 10961721344, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98809413632, + "reserved_bytes_current_delta": 59565408256, + "reserved_bytes_peak": 98809413632, + "reserved_bytes_peak_delta": 59565408256, + "segment_count_current": 438, + "segment_count_current_delta": 367, + "segment_count_peak": 438, + "segment_count_peak_delta": 367, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T130138175459Z.pickle", + "timestamp_utc": "2026-08-01T13:01:38.283348Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T130138175459Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T130138175459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..984eed35905040f4194c7fb89058cd02a9e0f3f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T130138175459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67b4b80b0a82135daca456cfbeaf4c045bd94310d1905824a23db70599b98150 +size 5385705 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T130139338807Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T130139338807Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1411fc751be55dbc1cfff135c97b2299832e931 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T130139338807Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115792877568, + "active_bytes_peak_delta": 23639744512, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115792877568, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115792877568, + "device": 2, + "inactive_split_bytes_current": 5036360704, + "inactive_split_bytes_peak": 11122107392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 119833362432, + "reserved_bytes_peak": 119833362432, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:01:39.338398Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115792877568, + "allocated_bytes_peak_delta": 23639744512, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92153133056, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92153133056, + "device": 2, + "inactive_split_bytes_current": 5252367360, + "inactive_split_bytes_peak": 11122107392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98809413632, + "reserved_bytes_peak": 98809413632, + "segment_count_current": 438, + "segment_count_peak": 438, + "timestamp_utc": "2026-08-01T13:01:38.300762Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5036360704, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 11122107392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 119833362432, + "reserved_bytes_current_delta": 21023948800, + "reserved_bytes_peak": 119833362432, + "reserved_bytes_peak_delta": 21023948800, + "segment_count_current": 492, + "segment_count_current_delta": 54, + "segment_count_peak": 492, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T130139338807Z.pickle", + "timestamp_utc": "2026-08-01T13:01:39.440541Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T130139338807Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T130139338807Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f4cfc25d4f32f0e2c4a6de9a6872beaf86475097 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T130139338807Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3504dfcb9eb8b61c6fdc920a473fdc7cb6601309d9b32998442a808da4f9a591 +size 5580281 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T130140501419Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T130140501419Z.json new file mode 100644 index 0000000000000000000000000000000000000000..821d20f8afaaa65b13512be4e9ccf674ee74c3c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T130140501419Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115801772032, + "active_bytes_peak_delta": 8894464, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115801772032, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115801772032, + "device": 2, + "inactive_split_bytes_current": 5034263552, + "inactive_split_bytes_peak": 11122107392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120892424192, + "reserved_bytes_peak": 120892424192, + "segment_count_current": 494, + "segment_count_peak": 494, + "timestamp_utc": "2026-08-01T13:01:40.501013Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115801772032, + "allocated_bytes_peak_delta": 8894464, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115792877568, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115792877568, + "device": 2, + "inactive_split_bytes_current": 5036360704, + "inactive_split_bytes_peak": 11122107392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 119833362432, + "reserved_bytes_peak": 119833362432, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:01:39.453398Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5034263552, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11122107392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120892424192, + "reserved_bytes_current_delta": 1059061760, + "reserved_bytes_peak": 120892424192, + "reserved_bytes_peak_delta": 1059061760, + "segment_count_current": 494, + "segment_count_current_delta": 2, + "segment_count_peak": 494, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T130140501419Z.pickle", + "timestamp_utc": "2026-08-01T13:01:40.604028Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T130140501419Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T130140501419Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff07023a85fadd26597836857be9e16290250369 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T130140501419Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9824ad6a7862c39a466127b623ef9f188ffa7a86d5970f089e22a1a80161a85a +size 5578989 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T130143261227Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T130143261227Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d3e6e28a503fd50ae3758fc57ce4918f8df1675 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T130143261227Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 167777986560, + "active_bytes_peak_delta": 51976214528, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167777986560, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115806995968, + "device": 2, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13966108672, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 181076492288, + "reserved_bytes_peak": 181076492288, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:43.260690Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115806995968, + "allocated_bytes_peak_delta": 5223936, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115801772032, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115801772032, + "device": 2, + "inactive_split_bytes_current": 5034263552, + "inactive_split_bytes_peak": 11122107392, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120892424192, + "reserved_bytes_peak": 120892424192, + "segment_count_current": 494, + "segment_count_peak": 494, + "timestamp_utc": "2026-08-01T13:01:40.616962Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_current_delta": 253489152, + "inactive_split_bytes_peak": 13966108672, + "inactive_split_bytes_peak_delta": 2844001280, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 181076492288, + "reserved_bytes_current_delta": 60184068096, + "reserved_bytes_peak": 181076492288, + "reserved_bytes_peak_delta": 60184068096, + "segment_count_current": 802, + "segment_count_current_delta": 308, + "segment_count_peak": 802, + "segment_count_peak_delta": 308, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T130143261227Z.pickle", + "timestamp_utc": "2026-08-01T13:01:43.377608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T130143261227Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T130143261227Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e87d5dce92e0a4b965850b325d980095c8980c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T130143261227Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3442f4d49e7557375e8e51c2297aed17ea2b44c70ded91243370712e4d6ec17 +size 6575174 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T130144227790Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T130144227790Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f8ccd400a55cc99e3e849c8e974c12482142134 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T130144227790Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167777986560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167777986560, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115806995968, + "device": 2, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13966108672, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 181076492288, + "reserved_bytes_peak": 181076492288, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:44.227319Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806995968, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167777986560, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115806995968, + "device": 2, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13966108672, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 181076492288, + "reserved_bytes_peak": 181076492288, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:43.392198Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13966108672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 181076492288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181076492288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T130144227790Z.pickle", + "timestamp_utc": "2026-08-01T13:01:44.234302Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T130144227790Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T130144227790Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..956d0912c59930130dcd3dcee30abd558f58e531 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T130144227790Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8798184a21eaa02f9a613453ece3d644c95a2ee8f80dbc21ddd1e794305cdbe +size 236645 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T130145065577Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T130145065577Z.json new file mode 100644 index 0000000000000000000000000000000000000000..47d54bd5ffa3af15fcc87ed3bf1844fe34356770 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T130145065577Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167777986560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167777986560, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115806995968, + "device": 2, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13966108672, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 181076492288, + "reserved_bytes_peak": 181076492288, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:45.065155Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806995968, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167777986560, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115806995968, + "device": 2, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13966108672, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 181076492288, + "reserved_bytes_peak": 181076492288, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:44.235239Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13966108672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 181076492288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181076492288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T130145065577Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.073692Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T130145065577Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T130145065577Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..97538517c47a84925f43be977dcea82bfd105362 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T130145065577Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5af184d9e4d073ee84825cd6a77fbe96be668eb3fec8bcc3006bbeb2524728bf +size 236614 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T130145901579Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T130145901579Z.json new file mode 100644 index 0000000000000000000000000000000000000000..633f109b2ab41471e336e683ff90e2a7de55fe47 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T130145901579Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167777986560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167777986560, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115806995968, + "device": 2, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13966108672, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 181076492288, + "reserved_bytes_peak": 181076492288, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:45.901199Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806995968, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167777986560, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115806995968, + "device": 2, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13966108672, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 181076492288, + "reserved_bytes_peak": 181076492288, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:45.074635Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13966108672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 181076492288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181076492288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T130145901579Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.907760Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T130145901579Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T130145901579Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ebaf0d1811fb8dc41bf4471674e38ae3de5dda26 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T130145901579Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44d1e1160293d2816ed5007fa855ee27d0cecf70fb597946ed6c9b16b42e20c3 +size 236645 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T130146731485Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T130146731485Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bf60ecc364abea8ae92eaee6e8bb19306bf17e8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T130146731485Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167777986560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167777986560, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115806995968, + "device": 2, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13966108672, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 181076492288, + "reserved_bytes_peak": 181076492288, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:46.731068Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806995968, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167777986560, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115806995968, + "device": 2, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13966108672, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 181076492288, + "reserved_bytes_peak": 181076492288, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:45.908665Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13966108672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 181076492288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181076492288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T130146731485Z.pickle", + "timestamp_utc": "2026-08-01T13:01:46.737477Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T130146731485Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T130146731485Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..01e6cfe836fd8c5ab88743b0019dbb1fa94c5837 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T130146731485Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d038447f9ed499b79921b0f5c6136129a5abe063745bee993bd0d168bba318d6 +size 236614 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T130147563596Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T130147563596Z.json new file mode 100644 index 0000000000000000000000000000000000000000..08b4f45b9835d45a6f496b92798e987185dacf03 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T130147563596Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167777986560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167777986560, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115806995968, + "device": 2, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13966108672, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 181076492288, + "reserved_bytes_peak": 181076492288, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:47.563199Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806995968, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167777986560, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115806995968, + "device": 2, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13966108672, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 181076492288, + "reserved_bytes_peak": 181076492288, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:46.738398Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13966108672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 181076492288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181076492288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T130147563596Z.pickle", + "timestamp_utc": "2026-08-01T13:01:47.569819Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T130147563596Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T130147563596Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6a78cee8647f174d2f24fb615ff55aa0a5bc2c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T130147563596Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dde5854cdb418f7f3fb4b8be23af9f5fa862dc9d743d7f30509dab26de7e5648 +size 236645 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T130138175513Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T130138175513Z.json new file mode 100644 index 0000000000000000000000000000000000000000..187685c2ee7e3d31ff8f269097351539e25e0442 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T130138175513Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 92159424512, + "active_bytes_peak_delta": 56693088256, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92159424512, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92159424512, + "device": 3, + "inactive_split_bytes_current": 5252367360, + "inactive_split_bytes_peak": 10769785856, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 98809413632, + "reserved_bytes_peak": 98809413632, + "segment_count_current": 438, + "segment_count_peak": 438, + "timestamp_utc": "2026-08-01T13:01:38.174922Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 92159424512, + "allocated_bytes_peak_delta": 56693088256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:00:29.416134Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5252367360, + "inactive_split_bytes_current_delta": 5119548416, + "inactive_split_bytes_peak": 10769785856, + "inactive_split_bytes_peak_delta": 10609399808, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 98809413632, + "reserved_bytes_current_delta": 59565408256, + "reserved_bytes_peak": 98809413632, + "reserved_bytes_peak_delta": 59565408256, + "segment_count_current": 438, + "segment_count_current_delta": 367, + "segment_count_peak": 438, + "segment_count_peak_delta": 367, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T130138175513Z.pickle", + "timestamp_utc": "2026-08-01T13:01:38.285337Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T130138175513Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T130138175513Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5f942e69581f1393906156bf1cdc37c3ce1a42dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T130138175513Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2106874268621c1658fb28887571bc6d08671a660e75528ba387ed53bfae8271 +size 5385705 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T130139338844Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T130139338844Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e96684fa5406bc22f5fa56229cb69b564606127d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T130139338844Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115792877568, + "active_bytes_peak_delta": 23633453056, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115792877568, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115792877568, + "device": 3, + "inactive_split_bytes_current": 5034263552, + "inactive_split_bytes_peak": 10769785856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119363600384, + "reserved_bytes_peak": 119363600384, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:01:39.338452Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115792877568, + "allocated_bytes_peak_delta": 23633453056, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92159424512, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92159424512, + "device": 3, + "inactive_split_bytes_current": 5252367360, + "inactive_split_bytes_peak": 10769785856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 98809413632, + "reserved_bytes_peak": 98809413632, + "segment_count_current": 438, + "segment_count_peak": 438, + "timestamp_utc": "2026-08-01T13:01:38.302836Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5034263552, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 10769785856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119363600384, + "reserved_bytes_current_delta": 20554186752, + "reserved_bytes_peak": 119363600384, + "reserved_bytes_peak_delta": 20554186752, + "segment_count_current": 490, + "segment_count_current_delta": 52, + "segment_count_peak": 490, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T130139338844Z.pickle", + "timestamp_utc": "2026-08-01T13:01:39.442207Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T130139338844Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T130139338844Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5cffdf2829837f98ae35ac8caf08e94ab89c1b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T130139338844Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f3baa1f15aca90070ebc79c21a259e1bf13bec229644bf302bacbdb1bd85ac4 +size 5579541 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T130140501421Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T130140501421Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d83bea441f2016c76b1ba1fb8f4ba4ca20e3f4c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T130140501421Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115801772032, + "active_bytes_peak_delta": 8894464, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115801772032, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115801772032, + "device": 3, + "inactive_split_bytes_current": 5269144576, + "inactive_split_bytes_peak": 10769785856, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120657543168, + "reserved_bytes_peak": 120657543168, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T13:01:40.501025Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115801772032, + "allocated_bytes_peak_delta": 8894464, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115792877568, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115792877568, + "device": 3, + "inactive_split_bytes_current": 5034263552, + "inactive_split_bytes_peak": 10769785856, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 119363600384, + "reserved_bytes_peak": 119363600384, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:01:39.455339Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5269144576, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 10769785856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120657543168, + "reserved_bytes_current_delta": 1293942784, + "reserved_bytes_peak": 120657543168, + "reserved_bytes_peak_delta": 1293942784, + "segment_count_current": 493, + "segment_count_current_delta": 3, + "segment_count_peak": 493, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T130140501421Z.pickle", + "timestamp_utc": "2026-08-01T13:01:40.601572Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T130140501421Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T130140501421Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d4fc3a0035ecd5b7a2563db2cd0157060cbe8514 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T130140501421Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adb0e43fb889525b5941dd65eb79d039cb50e186e2898afd2bd88bf90238b083 +size 5577043 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T130143261297Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T130143261297Z.json new file mode 100644 index 0000000000000000000000000000000000000000..43915a645d553b5a9907d2096bde2deb7a9d9989 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T130143261297Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 167773661184, + "active_bytes_peak_delta": 51971889152, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 3, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14168866304, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 181663694848, + "reserved_bytes_peak": 181663694848, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:43.260736Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115803325952, + "allocated_bytes_peak_delta": 1553920, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115801772032, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115801772032, + "device": 3, + "inactive_split_bytes_current": 5269144576, + "inactive_split_bytes_peak": 10769785856, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 120657543168, + "reserved_bytes_peak": 120657543168, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T13:01:40.614208Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_current_delta": 16510976, + "inactive_split_bytes_peak": 14168866304, + "inactive_split_bytes_peak_delta": 3399080448, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 181663694848, + "reserved_bytes_current_delta": 61006151680, + "reserved_bytes_peak": 181663694848, + "reserved_bytes_peak_delta": 61006151680, + "segment_count_current": 803, + "segment_count_current_delta": 310, + "segment_count_peak": 803, + "segment_count_peak_delta": 310, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T130143261297Z.pickle", + "timestamp_utc": "2026-08-01T13:01:43.379213Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T130143261297Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T130143261297Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f17008cfe7eb42a4aea0887c029723184aeecd0d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T130143261297Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:634039899e978ea4bea3b106edb7b88eba29e682a1541a9193179711f30bb447 +size 6574225 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T130144227768Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T130144227768Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1089d00596aff36219854c3d13321309c3eec6bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T130144227768Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167773661184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 3, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14168866304, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 181663694848, + "reserved_bytes_peak": 181663694848, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:44.227318Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115803325952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 3, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14168866304, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 181663694848, + "reserved_bytes_peak": 181663694848, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:43.393506Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14168866304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 181663694848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181663694848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 803, + "segment_count_current_delta": 0, + "segment_count_peak": 803, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T130144227768Z.pickle", + "timestamp_utc": "2026-08-01T13:01:44.233970Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T130144227768Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T130144227768Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..43277e18ec2cbfd82c8db32280abb754c2db3094 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T130144227768Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1898811f770fbaac135b1fca05fa85a1ef463c74f98b19433c213b9df90910c2 +size 235742 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T130145065520Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T130145065520Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ae2b97cbf9ca8772a9e4ba93df9da8b73e5f4fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T130145065520Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167773661184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 3, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14168866304, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 181663694848, + "reserved_bytes_peak": 181663694848, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:45.065102Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115803325952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 3, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14168866304, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 181663694848, + "reserved_bytes_peak": 181663694848, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:44.234899Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14168866304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 181663694848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181663694848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 803, + "segment_count_current_delta": 0, + "segment_count_peak": 803, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T130145065520Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.071768Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T130145065520Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T130145065520Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..74ff59e78fade894d554b7f2a7b2055f538285cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T130145065520Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f680a38a66258e3cc81655839041f4aa9a891e4ca658ec84a724d905c6fe061 +size 235711 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T130145901600Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T130145901600Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e8c772949f01a0ba6675087b8c3f5a246a5ec6e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T130145901600Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167773661184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 3, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14168866304, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 181663694848, + "reserved_bytes_peak": 181663694848, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:45.901211Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115803325952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 3, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14168866304, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 181663694848, + "reserved_bytes_peak": 181663694848, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:45.072691Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14168866304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 181663694848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181663694848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 803, + "segment_count_current_delta": 0, + "segment_count_peak": 803, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T130145901600Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.907246Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T130145901600Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T130145901600Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a98642ac2b2ef6bd4ad80f9a4d1b60e3fa150731 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T130145901600Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd64d63380c3353f6bbe4d8a5a10fed4644d25e66e649f05184b47f8ea18af4f +size 235742 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T130146731474Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T130146731474Z.json new file mode 100644 index 0000000000000000000000000000000000000000..73a4efef7685379428fbff05b5e16100ce7e74a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T130146731474Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167773661184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 3, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14168866304, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 181663694848, + "reserved_bytes_peak": 181663694848, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:46.731077Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115803325952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 3, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14168866304, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 181663694848, + "reserved_bytes_peak": 181663694848, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:45.908129Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14168866304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 181663694848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181663694848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 803, + "segment_count_current_delta": 0, + "segment_count_peak": 803, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T130146731474Z.pickle", + "timestamp_utc": "2026-08-01T13:01:46.737208Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T130146731474Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T130146731474Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1b4077ad0bb0bd622eff635648ddeb079734d1e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T130146731474Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d55936a987a2107917cfa3ad3c224394ec489e95b9e85c389f1e3fdc1cc11c52 +size 235711 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T130147563649Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T130147563649Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b2bae508bfdc63fd72bdecf14fb2e61a420e8e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T130147563649Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167773661184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 3, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14168866304, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 181663694848, + "reserved_bytes_peak": 181663694848, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:47.563255Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115803325952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 3, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14168866304, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 181663694848, + "reserved_bytes_peak": 181663694848, + "segment_count_current": 803, + "segment_count_peak": 803, + "timestamp_utc": "2026-08-01T13:01:46.738139Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14168866304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 181663694848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181663694848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 803, + "segment_count_current_delta": 0, + "segment_count_peak": 803, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T130147563649Z.pickle", + "timestamp_utc": "2026-08-01T13:01:47.569552Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T130147563649Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T130147563649Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e86855638195cbb8e502df6e11c70cacd8d3ca4c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T130147563649Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0130c6284b770970530c732658feb6f2392cc4defe3cf1a3a02a40894e6baaaf +size 235742 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T130138175501Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T130138175501Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c4a1c5c39d0b79829fa871ebe620f23723afd1a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T130138175501Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922842112, + "active_bytes_current_delta": 31690944512, + "active_bytes_peak": 92157894656, + "active_bytes_peak_delta": 56691116032, + "after": { + "active_bytes_current": 66922842112, + "active_bytes_peak": 92157894656, + "allocated_bytes_current": 66922842112, + "allocated_bytes_peak": 92157894656, + "device": 4, + "inactive_split_bytes_current": 5015762944, + "inactive_split_bytes_peak": 10844790784, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 98868133888, + "reserved_bytes_peak": 98868133888, + "segment_count_current": 439, + "segment_count_peak": 439, + "timestamp_utc": "2026-08-01T13:01:38.174920Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922842112, + "allocated_bytes_current_delta": 31690944512, + "allocated_bytes_peak": 92157894656, + "allocated_bytes_peak_delta": 56691116032, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:00:29.416497Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5015762944, + "inactive_split_bytes_current_delta": 4883386368, + "inactive_split_bytes_peak": 10844790784, + "inactive_split_bytes_peak_delta": 10691381248, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 98868133888, + "reserved_bytes_current_delta": 59624128512, + "reserved_bytes_peak": 98868133888, + "reserved_bytes_peak_delta": 59624128512, + "segment_count_current": 439, + "segment_count_current_delta": 368, + "segment_count_peak": 439, + "segment_count_peak_delta": 368, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T130138175501Z.pickle", + "timestamp_utc": "2026-08-01T13:01:38.286691Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T130138175501Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T130138175501Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..487e457d98ce54e8af95ffd03223fb3e83a26da5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T130138175501Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5748cc5cf85e73e0ca19ea64dfaa690dcc78729bb3147dbd8d0a866a23ccb027 +size 5393840 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T130139338817Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T130139338817Z.json new file mode 100644 index 0000000000000000000000000000000000000000..72220818a83a46e2761a6fea2fa3a33f93ce939c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T130139338817Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922842112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115801545728, + "active_bytes_peak_delta": 23643651072, + "after": { + "active_bytes_current": 66922842112, + "active_bytes_peak": 115801545728, + "allocated_bytes_current": 66922842112, + "allocated_bytes_peak": 115801545728, + "device": 4, + "inactive_split_bytes_current": 4803950592, + "inactive_split_bytes_peak": 10844790784, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 119952900096, + "reserved_bytes_peak": 119952900096, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:01:39.338413Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922842112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115801545728, + "allocated_bytes_peak_delta": 23643651072, + "before": { + "active_bytes_current": 66922842112, + "active_bytes_peak": 92157894656, + "allocated_bytes_current": 66922842112, + "allocated_bytes_peak": 92157894656, + "device": 4, + "inactive_split_bytes_current": 5015762944, + "inactive_split_bytes_peak": 10844790784, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 98868133888, + "reserved_bytes_peak": 98868133888, + "segment_count_current": 439, + "segment_count_peak": 439, + "timestamp_utc": "2026-08-01T13:01:38.304452Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4803950592, + "inactive_split_bytes_current_delta": -211812352, + "inactive_split_bytes_peak": 10844790784, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 119952900096, + "reserved_bytes_current_delta": 21084766208, + "reserved_bytes_peak": 119952900096, + "reserved_bytes_peak_delta": 21084766208, + "segment_count_current": 495, + "segment_count_current_delta": 56, + "segment_count_peak": 495, + "segment_count_peak_delta": 56, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T130139338817Z.pickle", + "timestamp_utc": "2026-08-01T13:01:39.441781Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T130139338817Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T130139338817Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13d32535cf08e9af1cd945bc307e6e1505e0068f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T130139338817Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b1c96018c613be5513597015b8eab1808ac08fc740cf2969d854331dc8e8fcb +size 5588752 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T130140501410Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T130140501410Z.json new file mode 100644 index 0000000000000000000000000000000000000000..38758fae5c98ff222958fc07f8b4f398ff90e96e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T130140501410Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66922842112, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115806787584, + "active_bytes_peak_delta": 5241856, + "after": { + "active_bytes_current": 66922842112, + "active_bytes_peak": 115806787584, + "allocated_bytes_current": 66922842112, + "allocated_bytes_peak": 115806787584, + "device": 4, + "inactive_split_bytes_current": 5032540160, + "inactive_split_bytes_peak": 10844790784, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121246842880, + "reserved_bytes_peak": 121246842880, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T13:01:40.501005Z", + "world_size": 8 + }, + "allocated_bytes_current": 66922842112, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115806787584, + "allocated_bytes_peak_delta": 5241856, + "before": { + "active_bytes_current": 66922842112, + "active_bytes_peak": 115801545728, + "allocated_bytes_current": 66922842112, + "allocated_bytes_peak": 115801545728, + "device": 4, + "inactive_split_bytes_current": 4803950592, + "inactive_split_bytes_peak": 10844790784, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 119952900096, + "reserved_bytes_peak": 119952900096, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:01:39.455331Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5032540160, + "inactive_split_bytes_current_delta": 228589568, + "inactive_split_bytes_peak": 10844790784, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121246842880, + "reserved_bytes_current_delta": 1293942784, + "reserved_bytes_peak": 121246842880, + "reserved_bytes_peak_delta": 1293942784, + "segment_count_current": 498, + "segment_count_current_delta": 3, + "segment_count_peak": 498, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T130140501410Z.pickle", + "timestamp_utc": "2026-08-01T13:01:40.608490Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T130140501410Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T130140501410Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a60a00a832647ceb22d667d5d747f455a50b210e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T130140501410Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be80a9dcb9e82958a7f184d73a24d4c0e0690c69ded3c68d1dde15f5cc638b82 +size 5586106 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T130143261372Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T130143261372Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef6362a6b23a8d502268e64ada6f21701d69b252 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T130143261372Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939885568, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 167777743872, + "active_bytes_peak_delta": 51970956288, + "after": { + "active_bytes_current": 66939885568, + "active_bytes_peak": 167777743872, + "allocated_bytes_current": 66939885568, + "allocated_bytes_peak": 115807408640, + "device": 4, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_peak": 13889930240, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 801, + "segment_count_peak": 801, + "timestamp_utc": "2026-08-01T13:01:43.260770Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939885568, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115807408640, + "allocated_bytes_peak_delta": 621056, + "before": { + "active_bytes_current": 66922842112, + "active_bytes_peak": 115806787584, + "allocated_bytes_current": 66922842112, + "allocated_bytes_peak": 115806787584, + "device": 4, + "inactive_split_bytes_current": 5032540160, + "inactive_split_bytes_peak": 10844790784, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121246842880, + "reserved_bytes_peak": 121246842880, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T13:01:40.624115Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_current_delta": 18608128, + "inactive_split_bytes_peak": 13889930240, + "inactive_split_bytes_peak_delta": 3045139456, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 60005810176, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 60005810176, + "segment_count_current": 801, + "segment_count_current_delta": 303, + "segment_count_peak": 801, + "segment_count_peak_delta": 303, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T130143261372Z.pickle", + "timestamp_utc": "2026-08-01T13:01:43.380503Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T130143261372Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T130143261372Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c006ffd033a015eeb83b035543e5f51b3177a07b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T130143261372Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c208d182e755b00d63ee434b2be6d1948adfe39633fd526c88c3fbdac21dc399 +size 6582918 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T130144227752Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T130144227752Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1910e9f69436384c6dff7102b07d38c21fc0d3c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T130144227752Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939885568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167777743872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939885568, + "active_bytes_peak": 167777743872, + "allocated_bytes_current": 66939885568, + "allocated_bytes_peak": 115807408640, + "device": 4, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_peak": 13889930240, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 801, + "segment_count_peak": 801, + "timestamp_utc": "2026-08-01T13:01:44.227276Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939885568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807408640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939885568, + "active_bytes_peak": 167777743872, + "allocated_bytes_current": 66939885568, + "allocated_bytes_peak": 115807408640, + "device": 4, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_peak": 13889930240, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 801, + "segment_count_peak": 801, + "timestamp_utc": "2026-08-01T13:01:43.395337Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13889930240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 801, + "segment_count_current_delta": 0, + "segment_count_peak": 801, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T130144227752Z.pickle", + "timestamp_utc": "2026-08-01T13:01:44.234052Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T130144227752Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T130144227752Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..557d4d92c89338496360e277fb4b1872fa5e4214 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T130144227752Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ac30b99d16b8149cb6ab692feee82094bfdd99391928a4e316fa8ac24f415f9 +size 244233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T130145065447Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T130145065447Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ee20d17f9acb682200064784add8b351251e776 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T130145065447Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939885568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167777743872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939885568, + "active_bytes_peak": 167777743872, + "allocated_bytes_current": 66939885568, + "allocated_bytes_peak": 115807408640, + "device": 4, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_peak": 13889930240, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 801, + "segment_count_peak": 801, + "timestamp_utc": "2026-08-01T13:01:45.065039Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939885568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807408640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939885568, + "active_bytes_peak": 167777743872, + "allocated_bytes_current": 66939885568, + "allocated_bytes_peak": 115807408640, + "device": 4, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_peak": 13889930240, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 801, + "segment_count_peak": 801, + "timestamp_utc": "2026-08-01T13:01:44.235097Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13889930240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 801, + "segment_count_current_delta": 0, + "segment_count_peak": 801, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T130145065447Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.071549Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T130145065447Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T130145065447Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c444b12fb13c7114d844e142330e27847000c0b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T130145065447Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e9d4d11cc151c1a848fde2f5d0962a8f010c6fbc9f52b7f32f566acb11059e7 +size 244233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T130145901542Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T130145901542Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ed7835a6bdaf7ccca4ad54c201b57a3c8205a1e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T130145901542Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939885568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167777743872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939885568, + "active_bytes_peak": 167777743872, + "allocated_bytes_current": 66939885568, + "allocated_bytes_peak": 115807408640, + "device": 4, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_peak": 13889930240, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 801, + "segment_count_peak": 801, + "timestamp_utc": "2026-08-01T13:01:45.901150Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939885568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807408640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939885568, + "active_bytes_peak": 167777743872, + "allocated_bytes_current": 66939885568, + "allocated_bytes_peak": 115807408640, + "device": 4, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_peak": 13889930240, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 801, + "segment_count_peak": 801, + "timestamp_utc": "2026-08-01T13:01:45.072481Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13889930240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 801, + "segment_count_current_delta": 0, + "segment_count_peak": 801, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T130145901542Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.907302Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T130145901542Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T130145901542Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea10abb0a0e6ae7846092d107c20082533005edc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T130145901542Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dbe23774929632f901da15fa69e8bc5019aac1891d19b4d017f35d7ed1b6f7e +size 244233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T130146731419Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T130146731419Z.json new file mode 100644 index 0000000000000000000000000000000000000000..736a68e115d098f69745ea4074c87080df650fa2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T130146731419Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939885568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167777743872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939885568, + "active_bytes_peak": 167777743872, + "allocated_bytes_current": 66939885568, + "allocated_bytes_peak": 115807408640, + "device": 4, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_peak": 13889930240, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 801, + "segment_count_peak": 801, + "timestamp_utc": "2026-08-01T13:01:46.731029Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939885568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807408640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939885568, + "active_bytes_peak": 167777743872, + "allocated_bytes_current": 66939885568, + "allocated_bytes_peak": 115807408640, + "device": 4, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_peak": 13889930240, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 801, + "segment_count_peak": 801, + "timestamp_utc": "2026-08-01T13:01:45.908236Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13889930240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 801, + "segment_count_current_delta": 0, + "segment_count_peak": 801, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T130146731419Z.pickle", + "timestamp_utc": "2026-08-01T13:01:46.737114Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T130146731419Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T130146731419Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..57dcc56f4ab8e79fc93897b0e9e39a06c8117047 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T130146731419Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bea347ea32579a4f188b1e4ecef4102cc4ae3929846a9dd60c2f2c998877fb9 +size 244233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T130147563565Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T130147563565Z.json new file mode 100644 index 0000000000000000000000000000000000000000..50e530534a3c8d05b9e7d12b7f2c39d4db017157 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T130147563565Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66939885568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167777743872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66939885568, + "active_bytes_peak": 167777743872, + "allocated_bytes_current": 66939885568, + "allocated_bytes_peak": 115807408640, + "device": 4, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_peak": 13889930240, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 801, + "segment_count_peak": 801, + "timestamp_utc": "2026-08-01T13:01:47.563175Z", + "world_size": 8 + }, + "allocated_bytes_current": 66939885568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807408640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66939885568, + "active_bytes_peak": 167777743872, + "allocated_bytes_current": 66939885568, + "allocated_bytes_peak": 115807408640, + "device": 4, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_peak": 13889930240, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 801, + "segment_count_peak": 801, + "timestamp_utc": "2026-08-01T13:01:46.738029Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5051148288, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13889930240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 801, + "segment_count_current_delta": 0, + "segment_count_peak": 801, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T130147563565Z.pickle", + "timestamp_utc": "2026-08-01T13:01:47.569411Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T130147563565Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T130147563565Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..300b1282eb00511871490c8d612e08de4452eef0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T130147563565Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e74c063634f2b832ee6deab79b77579cd53ac1e44a5710343b5add7097ea1814 +size 244233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T130138175500Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T130138175500Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b8b7137abb4cc9b68d78fdf8e592ca42f37a82ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T130138175500Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 92159424512, + "active_bytes_peak_delta": 56693088256, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92159424512, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92159424512, + "device": 5, + "inactive_split_bytes_current": 5252367360, + "inactive_split_bytes_peak": 10769785856, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 98809413632, + "reserved_bytes_peak": 98809413632, + "segment_count_current": 438, + "segment_count_peak": 438, + "timestamp_utc": "2026-08-01T13:01:38.174869Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 92159424512, + "allocated_bytes_peak_delta": 56693088256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:00:29.419125Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5252367360, + "inactive_split_bytes_current_delta": 5119548416, + "inactive_split_bytes_peak": 10769785856, + "inactive_split_bytes_peak_delta": 10609399808, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 98809413632, + "reserved_bytes_current_delta": 59565408256, + "reserved_bytes_peak": 98809413632, + "reserved_bytes_peak_delta": 59565408256, + "segment_count_current": 438, + "segment_count_current_delta": 367, + "segment_count_peak": 438, + "segment_count_peak_delta": 367, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T130138175500Z.pickle", + "timestamp_utc": "2026-08-01T13:01:38.286995Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T130138175500Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T130138175500Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6259b1f0a298d342dc9d968903cfb12b2cf806c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T130138175500Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19f78bb30cc5f9e15404b1e8753849ebb6ac0f279b4f3dbaec9afbe80eeb5b40 +size 5385700 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T130139338822Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T130139338822Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e57d4ec441ef2748f8e3d7427456242e2d320423 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T130139338822Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115792877568, + "active_bytes_peak_delta": 23633453056, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115792877568, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115792877568, + "device": 5, + "inactive_split_bytes_current": 5038457856, + "inactive_split_bytes_peak": 11112076288, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 119363600384, + "reserved_bytes_peak": 119363600384, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:01:39.338415Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115792877568, + "allocated_bytes_peak_delta": 23633453056, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92159424512, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92159424512, + "device": 5, + "inactive_split_bytes_current": 5252367360, + "inactive_split_bytes_peak": 10769785856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 98809413632, + "reserved_bytes_peak": 98809413632, + "segment_count_current": 438, + "segment_count_peak": 438, + "timestamp_utc": "2026-08-01T13:01:38.306325Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5038457856, + "inactive_split_bytes_current_delta": -213909504, + "inactive_split_bytes_peak": 11112076288, + "inactive_split_bytes_peak_delta": 342290432, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 119363600384, + "reserved_bytes_current_delta": 20554186752, + "reserved_bytes_peak": 119363600384, + "reserved_bytes_peak_delta": 20554186752, + "segment_count_current": 490, + "segment_count_current_delta": 52, + "segment_count_peak": 490, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T130139338822Z.pickle", + "timestamp_utc": "2026-08-01T13:01:39.445008Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T130139338822Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T130139338822Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2ce94f117139e84c972ff3fdd1f0e5cdbcf7c9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T130139338822Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fef0331493fc3f8fb0614025e28fefce1955af16f119d7db637a99b0c71b7edb +size 5579385 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T130140501421Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T130140501421Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d36110c55937497a40fb8cc5561d47f4515ff0de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T130140501421Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115801772032, + "active_bytes_peak_delta": 8894464, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115801772032, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115801772032, + "device": 5, + "inactive_split_bytes_current": 5267047424, + "inactive_split_bytes_peak": 11112076288, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120657543168, + "reserved_bytes_peak": 120657543168, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T13:01:40.501011Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115801772032, + "allocated_bytes_peak_delta": 8894464, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115792877568, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115792877568, + "device": 5, + "inactive_split_bytes_current": 5038457856, + "inactive_split_bytes_peak": 11112076288, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 119363600384, + "reserved_bytes_peak": 119363600384, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:01:39.460109Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5267047424, + "inactive_split_bytes_current_delta": 228589568, + "inactive_split_bytes_peak": 11112076288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120657543168, + "reserved_bytes_current_delta": 1293942784, + "reserved_bytes_peak": 120657543168, + "reserved_bytes_peak_delta": 1293942784, + "segment_count_current": 493, + "segment_count_current_delta": 3, + "segment_count_peak": 493, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T130140501421Z.pickle", + "timestamp_utc": "2026-08-01T13:01:40.606754Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T130140501421Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T130140501421Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..212f9214ea45ed0a766812a69c8e3135fbaee330 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T130140501421Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2bc023ed39738446362543848ee75358d05287d297ab7cd6bc2a6966de9aa3a +size 5577390 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T130143261221Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T130143261221Z.json new file mode 100644 index 0000000000000000000000000000000000000000..14ff8ec69dd110e983f2babd6259580ec68b7691 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T130143261221Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 167773661184, + "active_bytes_peak_delta": 51971889152, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 5, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14166769152, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 181661597696, + "reserved_bytes_peak": 181661597696, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:43.260700Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115803325952, + "allocated_bytes_peak_delta": 1553920, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115801772032, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115801772032, + "device": 5, + "inactive_split_bytes_current": 5267047424, + "inactive_split_bytes_peak": 11112076288, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120657543168, + "reserved_bytes_peak": 120657543168, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T13:01:40.621328Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_current_delta": 18608128, + "inactive_split_bytes_peak": 14166769152, + "inactive_split_bytes_peak_delta": 3054692864, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 181661597696, + "reserved_bytes_current_delta": 61004054528, + "reserved_bytes_peak": 181661597696, + "reserved_bytes_peak_delta": 61004054528, + "segment_count_current": 802, + "segment_count_current_delta": 309, + "segment_count_peak": 802, + "segment_count_peak_delta": 309, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T130143261221Z.pickle", + "timestamp_utc": "2026-08-01T13:01:43.380767Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T130143261221Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T130143261221Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ce5368cc8c62e5a9921a5e0badf21927c4475e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T130143261221Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c470475b49b0d3e1c3e2cfab1eeb2934d766425574e3d9c2a5b47f0e5627038d +size 6574392 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T130144227878Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T130144227878Z.json new file mode 100644 index 0000000000000000000000000000000000000000..68f029e825d62df33291c4e3e1ae3432c3bf2406 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T130144227878Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167773661184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 5, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14166769152, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 181661597696, + "reserved_bytes_peak": 181661597696, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:44.227400Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115803325952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 5, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14166769152, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 181661597696, + "reserved_bytes_peak": 181661597696, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:43.397510Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14166769152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 181661597696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181661597696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T130144227878Z.pickle", + "timestamp_utc": "2026-08-01T13:01:44.236805Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T130144227878Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T130144227878Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..25f06257c93bb44b0d67daef03600fc63829435a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T130144227878Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a709202c69c9b32dbf3bf2903087b83b862f0e31641c9aed6838d535e57d1ed7 +size 235658 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T130145065470Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T130145065470Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80ee3dad0d7cfcc29c58aad753b8cdf7202fb641 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T130145065470Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167773661184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 5, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14166769152, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 181661597696, + "reserved_bytes_peak": 181661597696, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:45.065063Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115803325952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 5, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14166769152, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 181661597696, + "reserved_bytes_peak": 181661597696, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:44.237964Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14166769152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 181661597696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181661597696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T130145065470Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.072020Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T130145065470Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T130145065470Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..667a29635cc1a982d11e37e0ee82c98694b5f5d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T130145065470Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:144700d290da7c2128aa89b1f37af0ea9ef71013c02410b50c417da26a12d72d +size 235658 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T130145901582Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T130145901582Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8eb6b2e7d3cf58d450fdfdb750f082abfc9973be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T130145901582Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167773661184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 5, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14166769152, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 181661597696, + "reserved_bytes_peak": 181661597696, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:45.901193Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115803325952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 5, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14166769152, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 181661597696, + "reserved_bytes_peak": 181661597696, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:45.072949Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14166769152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 181661597696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181661597696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T130145901582Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.907454Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T130145901582Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T130145901582Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b53fd389051651e2e5851445c96cbbfe009a1134 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T130145901582Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94cffa5d6a9012b67a459b39b82550110756df22c97c1528783bf7218c82729e +size 235658 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T130146731423Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T130146731423Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d7b3e5fe33ea3efb5c042a736594605c5a41bc16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T130146731423Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167773661184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 5, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14166769152, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 181661597696, + "reserved_bytes_peak": 181661597696, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:46.731027Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115803325952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 5, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14166769152, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 181661597696, + "reserved_bytes_peak": 181661597696, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:45.908378Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14166769152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 181661597696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181661597696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T130146731423Z.pickle", + "timestamp_utc": "2026-08-01T13:01:46.737119Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T130146731423Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T130146731423Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93e23368055e3b08439a28e9df89f6618bb53586 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T130146731423Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:894899e52bc82688e78f6cbb6d861f8af3cd2554873e4e92d94d33c371385bc8 +size 235658 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T130147563651Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T130147563651Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4e429e79471dabbebcba55972a9130fb4e330e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T130147563651Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167773661184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 5, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14166769152, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 181661597696, + "reserved_bytes_peak": 181661597696, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:47.563247Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115803325952, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167773661184, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115803325952, + "device": 5, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_peak": 14166769152, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 181661597696, + "reserved_bytes_peak": 181661597696, + "segment_count_current": 802, + "segment_count_peak": 802, + "timestamp_utc": "2026-08-01T13:01:46.738049Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5285655552, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14166769152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 181661597696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181661597696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 802, + "segment_count_current_delta": 0, + "segment_count_peak": 802, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T130147563651Z.pickle", + "timestamp_utc": "2026-08-01T13:01:47.569405Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T130147563651Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T130147563651Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..15007802bf2fb57a7e145e5683b8122510b9ddc1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T130147563651Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df3e1fb217efffbeb15d717609b43aa981b3eae5f34749c082562d5a9d373289 +size 235658 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T130138175655Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T130138175655Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f425241d86d194f2628faee66067dee460910379 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T130138175655Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 31689663488, + "active_bytes_peak": 92159424512, + "active_bytes_peak_delta": 56693088256, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92159424512, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92159424512, + "device": 6, + "inactive_split_bytes_current": 5252367360, + "inactive_split_bytes_peak": 10767688704, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98574532608, + "reserved_bytes_peak": 98574532608, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T13:01:38.175009Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 31689663488, + "allocated_bytes_peak": 92159424512, + "allocated_bytes_peak_delta": 56693088256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:00:29.416043Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5252367360, + "inactive_split_bytes_current_delta": 5119548416, + "inactive_split_bytes_peak": 10767688704, + "inactive_split_bytes_peak_delta": 10607302656, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98574532608, + "reserved_bytes_current_delta": 59330527232, + "reserved_bytes_peak": 98574532608, + "reserved_bytes_peak_delta": 59330527232, + "segment_count_current": 437, + "segment_count_current_delta": 366, + "segment_count_peak": 437, + "segment_count_peak_delta": 366, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T130138175655Z.pickle", + "timestamp_utc": "2026-08-01T13:01:38.285255Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T130138175655Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T130138175655Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a8851c8e901262bfc693afe3734cca3996c89de1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T130138175655Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:378d497fa5b81d5c472d5adf226adf13975c1afc56be295cd084201fdf704f7f +size 5384658 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T130139339227Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T130139339227Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4403d1dbef3f7c07e8f7ad31e9baad044bce7f81 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T130139339227Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115793401856, + "active_bytes_peak_delta": 23633977344, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115793401856, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115793401856, + "device": 6, + "inactive_split_bytes_current": 5036360704, + "inactive_split_bytes_peak": 10767688704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119128719360, + "reserved_bytes_peak": 119128719360, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T13:01:39.338802Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115793401856, + "allocated_bytes_peak_delta": 23633977344, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 92159424512, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 92159424512, + "device": 6, + "inactive_split_bytes_current": 5252367360, + "inactive_split_bytes_peak": 10767688704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98574532608, + "reserved_bytes_peak": 98574532608, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T13:01:38.301761Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5036360704, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 10767688704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119128719360, + "reserved_bytes_current_delta": 20554186752, + "reserved_bytes_peak": 119128719360, + "reserved_bytes_peak_delta": 20554186752, + "segment_count_current": 489, + "segment_count_current_delta": 52, + "segment_count_peak": 489, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T130139339227Z.pickle", + "timestamp_utc": "2026-08-01T13:01:39.441944Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T130139339227Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T130139339227Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..50e741803e30e66f552c21e1d88af37d7c81462a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T130139339227Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc10b9ecfeeddee65430e718e2ff8a603c6dbd92ff9f2b3f139a4cd7ed776582 +size 5579328 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T130140501507Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T130140501507Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bb7705122b84715f0d9edeab1897917382c50df8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T130140501507Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921118720, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115801772032, + "active_bytes_peak_delta": 8370176, + "after": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115801772032, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115801772032, + "device": 6, + "inactive_split_bytes_current": 5034263552, + "inactive_split_bytes_peak": 10767688704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 120187781120, + "reserved_bytes_peak": 120187781120, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:01:40.501103Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921118720, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115801772032, + "allocated_bytes_peak_delta": 8370176, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115793401856, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115793401856, + "device": 6, + "inactive_split_bytes_current": 5036360704, + "inactive_split_bytes_peak": 10767688704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 119128719360, + "reserved_bytes_peak": 119128719360, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T13:01:39.454019Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5034263552, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 10767688704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 120187781120, + "reserved_bytes_current_delta": 1059061760, + "reserved_bytes_peak": 120187781120, + "reserved_bytes_peak_delta": 1059061760, + "segment_count_current": 491, + "segment_count_current_delta": 2, + "segment_count_peak": 491, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T130140501507Z.pickle", + "timestamp_utc": "2026-08-01T13:01:40.603267Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T130140501507Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T130140501507Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e73f37664f4e85f426d42b6a2287da8e47276f67 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T130140501507Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed67d8963e1db1522aaf5b763dfb85d3c0c70e02e7c341157232a22f4172ff1b +size 5578464 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T130143261349Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T130143261349Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a0021cdc98db18359a51e1fedd0311b36c38dc10 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T130143261349Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 167784278016, + "active_bytes_peak_delta": 51982505984, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167784278016, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115809879552, + "device": 6, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13652326400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 800, + "segment_count_peak": 800, + "timestamp_utc": "2026-08-01T13:01:43.260817Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115809879552, + "allocated_bytes_peak_delta": 8107520, + "before": { + "active_bytes_current": 66921118720, + "active_bytes_peak": 115801772032, + "allocated_bytes_current": 66921118720, + "allocated_bytes_peak": 115801772032, + "device": 6, + "inactive_split_bytes_current": 5034263552, + "inactive_split_bytes_peak": 10767688704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 120187781120, + "reserved_bytes_peak": 120187781120, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:01:40.615230Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_current_delta": 253489152, + "inactive_split_bytes_peak": 13652326400, + "inactive_split_bytes_peak_delta": 2884637696, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 61064871936, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 61064871936, + "segment_count_current": 800, + "segment_count_current_delta": 309, + "segment_count_peak": 800, + "segment_count_peak_delta": 309, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T130143261349Z.pickle", + "timestamp_utc": "2026-08-01T13:01:43.377633Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T130143261349Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T130143261349Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7efb733c84cbbc5c700a4793d11e4244ae9922ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T130143261349Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4879fa42a72247f0d4d3f6042271f52ef1bc8c7716316eb6e7040ba53463edb +size 6574799 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T130144227830Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T130144227830Z.json new file mode 100644 index 0000000000000000000000000000000000000000..379dafb0bfb798b6f624a21717a503969f5fcd9e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T130144227830Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167784278016, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167784278016, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115809879552, + "device": 6, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13652326400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 800, + "segment_count_peak": 800, + "timestamp_utc": "2026-08-01T13:01:44.227386Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115809879552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167784278016, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115809879552, + "device": 6, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13652326400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 800, + "segment_count_peak": 800, + "timestamp_utc": "2026-08-01T13:01:43.390971Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13652326400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 800, + "segment_count_current_delta": 0, + "segment_count_peak": 800, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T130144227830Z.pickle", + "timestamp_utc": "2026-08-01T13:01:44.233719Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T130144227830Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T130144227830Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..05aaa2c816f571b60fa876d850390110a4110568 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T130144227830Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:673663766182ca71f2f94560c9ee7b8553e21ec96e6bf4805a6b7833434a283c +size 236090 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T130145065565Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T130145065565Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd45f59693ce2404d3791bb24a0942cd33cd3613 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T130145065565Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167784278016, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167784278016, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115809879552, + "device": 6, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13652326400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 800, + "segment_count_peak": 800, + "timestamp_utc": "2026-08-01T13:01:45.065157Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115809879552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167784278016, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115809879552, + "device": 6, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13652326400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 800, + "segment_count_peak": 800, + "timestamp_utc": "2026-08-01T13:01:44.234717Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13652326400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 800, + "segment_count_current_delta": 0, + "segment_count_peak": 800, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T130145065565Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.071624Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T130145065565Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T130145065565Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6e8b159de044c455d43d2dd935c93a4b14540e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T130145065565Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f900a8ca235a14e1c33d209f22c9bd472531f88e91726267986b9fdb6e80ce3 +size 236090 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T130145901663Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T130145901663Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bbda12bbc6c8689c5a7ed87d1db63a497da6552a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T130145901663Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167784278016, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167784278016, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115809879552, + "device": 6, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13652326400, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 800, + "segment_count_peak": 800, + "timestamp_utc": "2026-08-01T13:01:45.901271Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115809879552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167784278016, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115809879552, + "device": 6, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13652326400, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 800, + "segment_count_peak": 800, + "timestamp_utc": "2026-08-01T13:01:45.072559Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13652326400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 800, + "segment_count_current_delta": 0, + "segment_count_peak": 800, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T130145901663Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.907334Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T130145901663Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T130145901663Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..47e7067738057823e4ab1ed79986914bb7ef9b86 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T130145901663Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f664601c968333fa96e5c6b413f157aeb57e5919aaf98b6d8e4e721fe747743e +size 236090 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T130146731619Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T130146731619Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f8aa5f517d243b20e7d0e8a969bc6d89b5c1b36 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T130146731619Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167784278016, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167784278016, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115809879552, + "device": 6, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13652326400, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 800, + "segment_count_peak": 800, + "timestamp_utc": "2026-08-01T13:01:46.731210Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115809879552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167784278016, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115809879552, + "device": 6, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13652326400, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 800, + "segment_count_peak": 800, + "timestamp_utc": "2026-08-01T13:01:45.908263Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13652326400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 800, + "segment_count_current_delta": 0, + "segment_count_peak": 800, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T130146731619Z.pickle", + "timestamp_utc": "2026-08-01T13:01:46.737434Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T130146731619Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T130146731619Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..759322ba1caeabdaa6422eec8befa7c1dce95769 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T130146731619Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5046332e24543bff8f9e4cc4468e0f9276fe6cee3ee5e5a49038c25d6a43bb1a +size 236090 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T130147563689Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T130147563689Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd0fcbbead73e1cfb1d5485bab1e67287ca54d7f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T130147563689Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162176, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167784278016, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167784278016, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115809879552, + "device": 6, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13652326400, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 800, + "segment_count_peak": 800, + "timestamp_utc": "2026-08-01T13:01:47.563287Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162176, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115809879552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162176, + "active_bytes_peak": 167784278016, + "allocated_bytes_current": 66938162176, + "allocated_bytes_peak": 115809879552, + "device": 6, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_peak": 13652326400, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 181252653056, + "reserved_bytes_peak": 181252653056, + "segment_count_current": 800, + "segment_count_peak": 800, + "timestamp_utc": "2026-08-01T13:01:46.738375Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5287752704, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 13652326400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 181252653056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181252653056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 800, + "segment_count_current_delta": 0, + "segment_count_peak": 800, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T130147563689Z.pickle", + "timestamp_utc": "2026-08-01T13:01:47.569672Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T130147563689Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T130147563689Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca9f47c691d55e5fe2c5396a9bed448bb36f4dd0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T130147563689Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e50b33f554d32a5a8db1bd0dde56920797d6eaa72bef8e91275ee5bbf5c84b84 +size 236090 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T130138175403Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T130138175403Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee5c33f7aadecc038bc8f5e697ce66fc991ce6f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T130138175403Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 31689664000, + "active_bytes_peak": 92153133056, + "active_bytes_peak_delta": 56686796800, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 92153133056, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 92153133056, + "device": 7, + "inactive_split_bytes_current": 5487247872, + "inactive_split_bytes_peak": 11122107392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 98809413632, + "reserved_bytes_peak": 98809413632, + "segment_count_current": 438, + "segment_count_peak": 438, + "timestamp_utc": "2026-08-01T13:01:38.174891Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 31689664000, + "allocated_bytes_peak": 92153133056, + "allocated_bytes_peak_delta": 56686796800, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:00:29.419858Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5487247872, + "inactive_split_bytes_current_delta": 5354428928, + "inactive_split_bytes_peak": 11122107392, + "inactive_split_bytes_peak_delta": 10961721344, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 98809413632, + "reserved_bytes_current_delta": 59565408256, + "reserved_bytes_peak": 98809413632, + "reserved_bytes_peak_delta": 59565408256, + "segment_count_current": 438, + "segment_count_current_delta": 367, + "segment_count_peak": 438, + "segment_count_peak_delta": 367, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T130138175403Z.pickle", + "timestamp_utc": "2026-08-01T13:01:38.286287Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T130138175403Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T130138175403Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..40ff29b22a15ec1da69f5526d3d27a3f0311ba58 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T130138175403Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23784fea466d2f1ce6bc3c332f531e9ef89ebfbe45b43edf3e10c73af0954612 +size 5387356 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T130139338834Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T130139338834Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dc10a382cdc1c3ea37d220a0723bdde70913cfe5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T130139338834Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115792878080, + "active_bytes_peak_delta": 23639745024, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 115792878080, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 115792878080, + "device": 7, + "inactive_split_bytes_current": 5040554496, + "inactive_split_bytes_peak": 11122107392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119363600384, + "reserved_bytes_peak": 119363600384, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:01:39.338432Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115792878080, + "allocated_bytes_peak_delta": 23639745024, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 92153133056, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 92153133056, + "device": 7, + "inactive_split_bytes_current": 5487247872, + "inactive_split_bytes_peak": 11122107392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 98809413632, + "reserved_bytes_peak": 98809413632, + "segment_count_current": 438, + "segment_count_peak": 438, + "timestamp_utc": "2026-08-01T13:01:38.305797Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5040554496, + "inactive_split_bytes_current_delta": -446693376, + "inactive_split_bytes_peak": 11122107392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119363600384, + "reserved_bytes_current_delta": 20554186752, + "reserved_bytes_peak": 119363600384, + "reserved_bytes_peak_delta": 20554186752, + "segment_count_current": 490, + "segment_count_current_delta": 52, + "segment_count_peak": 490, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T130139338834Z.pickle", + "timestamp_utc": "2026-08-01T13:01:39.443964Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T130139338834Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T130139338834Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..436691a70cb43ad0478573f5f3f9305698116c12 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T130139338834Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:953b18e564627f4ec474e96285591afae7e39886e8f0e1f1782b2b0554578e7d +size 5581258 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T130140501440Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T130140501440Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ce3b4eb64b77d579fe1cada280ce93e945fef364 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T130140501440Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66921119232, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115801772544, + "active_bytes_peak_delta": 8894464, + "after": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 115801772544, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 115801772544, + "device": 7, + "inactive_split_bytes_current": 5273338368, + "inactive_split_bytes_peak": 11122107392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 120657543168, + "reserved_bytes_peak": 120657543168, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T13:01:40.501035Z", + "world_size": 8 + }, + "allocated_bytes_current": 66921119232, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115801772544, + "allocated_bytes_peak_delta": 8894464, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 115792878080, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 115792878080, + "device": 7, + "inactive_split_bytes_current": 5040554496, + "inactive_split_bytes_peak": 11122107392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119363600384, + "reserved_bytes_peak": 119363600384, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:01:39.459540Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5273338368, + "inactive_split_bytes_current_delta": 232783872, + "inactive_split_bytes_peak": 11122107392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 120657543168, + "reserved_bytes_current_delta": 1293942784, + "reserved_bytes_peak": 120657543168, + "reserved_bytes_peak_delta": 1293942784, + "segment_count_current": 493, + "segment_count_current_delta": 3, + "segment_count_peak": 493, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T130140501440Z.pickle", + "timestamp_utc": "2026-08-01T13:01:40.605440Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T130140501440Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T130140501440Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ac5d8a2dfe63255aabf8c2384a94ec7cdd9a51e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T130140501440Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbd4e9b1e8350e2fcb7736cc064dc53e12f3fd65790c1acdbffebc6d22b7e088 +size 5580139 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T130143261303Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T130143261303Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70aa767895af95db2ea1ad2dda8c36ad0e5d9c43 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T130143261303Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 17043456, + "active_bytes_peak": 167783229952, + "active_bytes_peak_delta": 51981457408, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_peak": 14156710400, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 804, + "segment_count_peak": 804, + "timestamp_utc": "2026-08-01T13:01:43.260778Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 17043456, + "allocated_bytes_peak": 115807652352, + "allocated_bytes_peak_delta": 5879808, + "before": { + "active_bytes_current": 66921119232, + "active_bytes_peak": 115801772544, + "allocated_bytes_current": 66921119232, + "allocated_bytes_peak": 115801772544, + "device": 7, + "inactive_split_bytes_current": 5273338368, + "inactive_split_bytes_peak": 11122107392, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 120657543168, + "reserved_bytes_peak": 120657543168, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T13:01:40.620307Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_current_delta": 16510976, + "inactive_split_bytes_peak": 14156710400, + "inactive_split_bytes_peak_delta": 3034603008, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 181491728384, + "reserved_bytes_current_delta": 60834185216, + "reserved_bytes_peak": 181491728384, + "reserved_bytes_peak_delta": 60834185216, + "segment_count_current": 804, + "segment_count_current_delta": 311, + "segment_count_peak": 804, + "segment_count_peak_delta": 311, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T130143261303Z.pickle", + "timestamp_utc": "2026-08-01T13:01:43.380449Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T130143261303Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T130143261303Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3292c2216dfe64ef08a063824da8661bd4f1cff7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T130143261303Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb1e3340a53f9458ec4e807dd68e8576e05b1d5e7bd04929eac53a2ec52b591b +size 6576340 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T130144227765Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T130144227765Z.json new file mode 100644 index 0000000000000000000000000000000000000000..381986de95c2383634c5096cf8e7ba0a5fa95782 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T130144227765Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167783229952, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_peak": 14156710400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 804, + "segment_count_peak": 804, + "timestamp_utc": "2026-08-01T13:01:44.227305Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807652352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_peak": 14156710400, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 804, + "segment_count_peak": 804, + "timestamp_utc": "2026-08-01T13:01:43.396810Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14156710400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 181491728384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181491728384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 804, + "segment_count_current_delta": 0, + "segment_count_peak": 804, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T130144227765Z.pickle", + "timestamp_utc": "2026-08-01T13:01:44.234231Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T130144227765Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T130144227765Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0f7574d7c47e66356492dc6543e0cde20805518 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T130144227765Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffeea3e1ffa40927da47bb815df6379eb54e5440128a28a901704153d124569f +size 238028 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T130145065518Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T130145065518Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d907d0c81fbf2fcb164619900cf61e33540718e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T130145065518Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167783229952, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_peak": 14156710400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 804, + "segment_count_peak": 804, + "timestamp_utc": "2026-08-01T13:01:45.065109Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807652352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_peak": 14156710400, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 804, + "segment_count_peak": 804, + "timestamp_utc": "2026-08-01T13:01:44.235152Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14156710400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 181491728384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181491728384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 804, + "segment_count_current_delta": 0, + "segment_count_peak": 804, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T130145065518Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.071574Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T130145065518Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T130145065518Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c58b6589ca1ba808a70fe2a6781f01188feed00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T130145065518Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02220e383b006f94dafd841cc62ea5c82a37e66fea680c85d198260bc366c074 +size 237997 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T130145901629Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T130145901629Z.json new file mode 100644 index 0000000000000000000000000000000000000000..444c1e334abd87ba5aa078d30c3b27fdd1120ea0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T130145901629Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167783229952, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_peak": 14156710400, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 804, + "segment_count_peak": 804, + "timestamp_utc": "2026-08-01T13:01:45.901243Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807652352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_peak": 14156710400, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 804, + "segment_count_peak": 804, + "timestamp_utc": "2026-08-01T13:01:45.072488Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14156710400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 181491728384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181491728384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 804, + "segment_count_current_delta": 0, + "segment_count_peak": 804, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T130145901629Z.pickle", + "timestamp_utc": "2026-08-01T13:01:45.907408Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T130145901629Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T130145901629Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d77ccf8df8d56754bd651575739e0f5dc6c14bda --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T130145901629Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:348b747668b32b5ed95fdecfce1f896fa29b68fae084ab9099ee240d637f4f79 +size 238028 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T130146731496Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T130146731496Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9cc13b116d1dd032c1fc54ede909f02ce5080e06 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T130146731496Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167783229952, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_peak": 14156710400, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 804, + "segment_count_peak": 804, + "timestamp_utc": "2026-08-01T13:01:46.731105Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807652352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_peak": 14156710400, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 804, + "segment_count_peak": 804, + "timestamp_utc": "2026-08-01T13:01:45.908304Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14156710400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 181491728384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181491728384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 804, + "segment_count_current_delta": 0, + "segment_count_peak": 804, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T130146731496Z.pickle", + "timestamp_utc": "2026-08-01T13:01:46.737206Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T130146731496Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T130146731496Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d32d246fef5d64af763c2155c7cdb8365d769ea8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T130146731496Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eef0339d0356705221b7e68d43dbf9ddde9c85c210b359d0b8c4c38b907029df +size 238028 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T130147563718Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T130147563718Z.json new file mode 100644 index 0000000000000000000000000000000000000000..553fe06aca298297a91d847a94e963269acd6054 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T130147563718Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66938162688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 167783229952, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_peak": 14156710400, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 804, + "segment_count_peak": 804, + "timestamp_utc": "2026-08-01T13:01:47.563289Z", + "world_size": 8 + }, + "allocated_bytes_current": 66938162688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115807652352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_peak": 14156710400, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 804, + "segment_count_peak": 804, + "timestamp_utc": "2026-08-01T13:01:46.738093Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 14156710400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 181491728384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 181491728384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 804, + "segment_count_current_delta": 0, + "segment_count_peak": 804, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T130147563718Z.pickle", + "timestamp_utc": "2026-08-01T13:01:47.569520Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T130147563718Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T130147563718Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba0100b4f8b9bd8f89422136b245d2ea3739ba66 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T130147563718Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9675c1dea3af4b3350e77e806c85e2f32921e31bb032ec0969354404fa551fb1 +size 237997 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T130149831897Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T130149831897Z.json new file mode 100644 index 0000000000000000000000000000000000000000..79b55f171f4df5c0f619ee5070bd4f6446808935 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T130149831897Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66955108864, + "active_bytes_current_delta": 16946176, + "active_bytes_peak": 167783229952, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66955108864, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66955108864, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5027536384, + "inactive_split_bytes_peak": 14156710400, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 6, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 182760505344, + "reserved_bytes_peak": 182760505344, + "segment_count_current": 767, + "segment_count_peak": 810, + "timestamp_utc": "2026-08-01T13:01:49.831492Z", + "world_size": 8 + }, + "allocated_bytes_current": 66955108864, + "allocated_bytes_current_delta": 16946176, + "allocated_bytes_peak": 115807652352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66938162688, + "active_bytes_peak": 167783229952, + "allocated_bytes_current": 66938162688, + "allocated_bytes_peak": 115807652352, + "device": 7, + "inactive_split_bytes_current": 5289849344, + "inactive_split_bytes_peak": 14156710400, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 181491728384, + "reserved_bytes_peak": 181491728384, + "segment_count_current": 804, + "segment_count_peak": 804, + "timestamp_utc": "2026-08-01T13:01:47.570460Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5027536384, + "inactive_split_bytes_current_delta": -262312960, + "inactive_split_bytes_peak": 14156710400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 6, + "num_alloc_retries_delta": 6, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 182760505344, + "reserved_bytes_current_delta": 1268776960, + "reserved_bytes_peak": 182760505344, + "reserved_bytes_peak_delta": 1268776960, + "segment_count_current": 767, + "segment_count_current_delta": -37, + "segment_count_peak": 810, + "segment_count_peak_delta": 6, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T130149831897Z.pickle", + "timestamp_utc": "2026-08-01T13:01:49.901932Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T130149831897Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T130149831897Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3f869976a140f58e073649547dcefaf47704dd9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T130149831897Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22a0d53845ec0a1cc0fecc216af812ace732028c52aae19c20e2c03ce27bd5a1 +size 3901554 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T131423399516Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T131423399516Z.json new file mode 100644 index 0000000000000000000000000000000000000000..20e4bf851c8829ca3513faf9a81af709db5748ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T131423399516Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508128256, + "active_bytes_current_delta": 37276673024, + "active_bytes_peak": 92748283904, + "active_bytes_peak_delta": 57281947648, + "after": { + "active_bytes_current": 72508128256, + "active_bytes_peak": 92748283904, + "allocated_bytes_current": 72508128256, + "allocated_bytes_peak": 92043640832, + "device": 0, + "inactive_split_bytes_current": 6481101824, + "inactive_split_bytes_peak": 11416422912, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98167685120, + "reserved_bytes_peak": 98167685120, + "segment_count_current": 379, + "segment_count_peak": 379, + "timestamp_utc": "2026-08-01T13:14:23.398931Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508128256, + "allocated_bytes_current_delta": 37276673024, + "allocated_bytes_peak": 92043640832, + "allocated_bytes_peak_delta": 56577304576, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:13:14.705743Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6481101824, + "inactive_split_bytes_current_delta": 6346185728, + "inactive_split_bytes_peak": 11416422912, + "inactive_split_bytes_peak_delta": 11255791104, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98167685120, + "reserved_bytes_current_delta": 58921582592, + "reserved_bytes_peak": 98167685120, + "reserved_bytes_peak_delta": 58921582592, + "segment_count_current": 379, + "segment_count_current_delta": 307, + "segment_count_peak": 379, + "segment_count_peak_delta": 307, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T131423399516Z.pickle", + "timestamp_utc": "2026-08-01T13:14:23.517044Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T131423399516Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T131423399516Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..00501e16d519acd905f95e623d86dd6491618fe0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T131423399516Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:882eddc483f4b5babe32e585dbd44ef2c5a65ffd44aef6c4f7537457cfae2ee1 +size 5952693 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T131424784355Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T131424784355Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2dd0e1bd9017ad81752f5da1940db81ec7d2d4a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T131424784355Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508128256, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116403071488, + "active_bytes_peak_delta": 23654787584, + "after": { + "active_bytes_current": 72508128256, + "active_bytes_peak": 116403071488, + "allocated_bytes_current": 72508128256, + "allocated_bytes_peak": 115704416256, + "device": 0, + "inactive_split_bytes_current": 9542943744, + "inactive_split_bytes_peak": 14511770624, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 121234259968, + "reserved_bytes_peak": 121234259968, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T13:14:24.783931Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508128256, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115704416256, + "allocated_bytes_peak_delta": 23660775424, + "before": { + "active_bytes_current": 72508128256, + "active_bytes_peak": 92748283904, + "allocated_bytes_current": 72508128256, + "allocated_bytes_peak": 92043640832, + "device": 0, + "inactive_split_bytes_current": 6481101824, + "inactive_split_bytes_peak": 11416422912, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98167685120, + "reserved_bytes_peak": 98167685120, + "segment_count_current": 379, + "segment_count_peak": 379, + "timestamp_utc": "2026-08-01T13:14:23.535473Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9542943744, + "inactive_split_bytes_current_delta": 3061841920, + "inactive_split_bytes_peak": 14511770624, + "inactive_split_bytes_peak_delta": 3095347712, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 121234259968, + "reserved_bytes_current_delta": 23066574848, + "reserved_bytes_peak": 121234259968, + "reserved_bytes_peak_delta": 23066574848, + "segment_count_current": 442, + "segment_count_current_delta": 63, + "segment_count_peak": 442, + "segment_count_peak_delta": 63, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T131424784355Z.pickle", + "timestamp_utc": "2026-08-01T13:14:24.894249Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T131424784355Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T131424784355Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0dbb7e0424af93f7c350abf444b188cb0e1aff19 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T131424784355Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc37b4ece815113ddfc9fb7f05d127b5cadd8f11bd5808b56346e439e8058ace +size 6090417 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T131426102012Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T131426102012Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d6abcbdf286fa7f116346de223448eeceba7573 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T131426102012Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72508128256, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116767823872, + "active_bytes_peak_delta": 364752384, + "after": { + "active_bytes_current": 72508128256, + "active_bytes_peak": 116767823872, + "allocated_bytes_current": 72508128256, + "allocated_bytes_peak": 115710859264, + "device": 0, + "inactive_split_bytes_current": 8251098112, + "inactive_split_bytes_peak": 14710576128, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 121234259968, + "reserved_bytes_peak": 121234259968, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T13:14:26.101602Z", + "world_size": 8 + }, + "allocated_bytes_current": 72508128256, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115710859264, + "allocated_bytes_peak_delta": 6443008, + "before": { + "active_bytes_current": 72508128256, + "active_bytes_peak": 116403071488, + "allocated_bytes_current": 72508128256, + "allocated_bytes_peak": 115704416256, + "device": 0, + "inactive_split_bytes_current": 9542943744, + "inactive_split_bytes_peak": 14511770624, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 121234259968, + "reserved_bytes_peak": 121234259968, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T13:14:24.908702Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8251098112, + "inactive_split_bytes_current_delta": -1291845632, + "inactive_split_bytes_peak": 14710576128, + "inactive_split_bytes_peak_delta": 198805504, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 121234259968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121234259968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 442, + "segment_count_current_delta": 0, + "segment_count_peak": 442, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T131426102012Z.pickle", + "timestamp_utc": "2026-08-01T13:14:26.211665Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T131426102012Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T131426102012Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b4d6fba62c60bf9f1d0d761e54f05d0d14283f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T131426102012Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb5c9e7f0dd08a7068f23cd4bc6fadcb7d0cc19cae13ab04a4fe8d51b30f82a7 +size 6609226 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T131427764419Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T131427764419Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2cac804fa7f384e40d23d1ff416cdc80dcddd51b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T131427764419Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 164324967424, + "active_bytes_current_delta": 91816839168, + "active_bytes_peak": 164324967424, + "active_bytes_peak_delta": 47557143552, + "after": { + "active_bytes_current": 164324967424, + "active_bytes_peak": 164324967424, + "allocated_bytes_current": 108120482816, + "allocated_bytes_peak": 115710859264, + "device": 0, + "inactive_split_bytes_current": 12783713280, + "inactive_split_bytes_peak": 19150849024, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 182804545536, + "reserved_bytes_peak": 182804545536, + "segment_count_current": 674, + "segment_count_peak": 674, + "timestamp_utc": "2026-08-01T13:14:27.763833Z", + "world_size": 8 + }, + "allocated_bytes_current": 108120482816, + "allocated_bytes_current_delta": 35612354560, + "allocated_bytes_peak": 115710859264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72508128256, + "active_bytes_peak": 116767823872, + "allocated_bytes_current": 72508128256, + "allocated_bytes_peak": 115710859264, + "device": 0, + "inactive_split_bytes_current": 8251098112, + "inactive_split_bytes_peak": 14710576128, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 121234259968, + "reserved_bytes_peak": 121234259968, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T13:14:26.226236Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12783713280, + "inactive_split_bytes_current_delta": 4532615168, + "inactive_split_bytes_peak": 19150849024, + "inactive_split_bytes_peak_delta": 4440272896, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 182804545536, + "reserved_bytes_current_delta": 61570285568, + "reserved_bytes_peak": 182804545536, + "reserved_bytes_peak_delta": 61570285568, + "segment_count_current": 674, + "segment_count_current_delta": 232, + "segment_count_peak": 674, + "segment_count_peak_delta": 232, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T131427764419Z.pickle", + "timestamp_utc": "2026-08-01T13:14:27.870750Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T131427764419Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T131427764419Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..41ede2676d14934f18f988c7d567d9a938eaf036 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T131427764419Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f476b58383aa8e572311dcf49a933f97b3a6ae30efa9e2051b66ca16597735d +size 5525999 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T131423399459Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T131423399459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1aaeb614b335589daabd535687892a0976ac1fb3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T131423399459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 92756920320, + "active_bytes_peak_delta": 57290584064, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92756920320, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92052277248, + "device": 1, + "inactive_split_bytes_current": 7127532544, + "inactive_split_bytes_peak": 12204380160, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 98931048448, + "reserved_bytes_peak": 98931048448, + "segment_count_current": 378, + "segment_count_peak": 378, + "timestamp_utc": "2026-08-01T13:14:23.398858Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 92052277248, + "allocated_bytes_peak_delta": 56585940992, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:13:14.705867Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7127532544, + "inactive_split_bytes_current_delta": 6994713600, + "inactive_split_bytes_peak": 12204380160, + "inactive_split_bytes_peak_delta": 12043994112, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 98931048448, + "reserved_bytes_current_delta": 59687043072, + "reserved_bytes_peak": 98931048448, + "reserved_bytes_peak_delta": 59687043072, + "segment_count_current": 378, + "segment_count_current_delta": 307, + "segment_count_peak": 378, + "segment_count_peak_delta": 307, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T131423399459Z.pickle", + "timestamp_utc": "2026-08-01T13:14:23.519566Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T131423399459Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T131423399459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e06ad963ea8f2c6ed587ae804b82f419b1eb13f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T131423399459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab0ed3d3e743c2babe008db363e5f3fdbc459501624496c105851790c6aa2a27 +size 5943290 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T131424784358Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T131424784358Z.json new file mode 100644 index 0000000000000000000000000000000000000000..acdd5e3b41dd8b892207ecd543895d8be476a535 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T131424784358Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116391787520, + "active_bytes_peak_delta": 23634867200, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116391787520, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115697780736, + "device": 1, + "inactive_split_bytes_current": 9595880448, + "inactive_split_bytes_peak": 17136631808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 121597067264, + "reserved_bytes_peak": 121597067264, + "segment_count_current": 436, + "segment_count_peak": 436, + "timestamp_utc": "2026-08-01T13:14:24.783932Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115697780736, + "allocated_bytes_peak_delta": 23645503488, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92756920320, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92052277248, + "device": 1, + "inactive_split_bytes_current": 7127532544, + "inactive_split_bytes_peak": 12204380160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 98931048448, + "reserved_bytes_peak": 98931048448, + "segment_count_current": 378, + "segment_count_peak": 378, + "timestamp_utc": "2026-08-01T13:14:23.538195Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9595880448, + "inactive_split_bytes_current_delta": 2468347904, + "inactive_split_bytes_peak": 17136631808, + "inactive_split_bytes_peak_delta": 4932251648, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 121597067264, + "reserved_bytes_current_delta": 22666018816, + "reserved_bytes_peak": 121597067264, + "reserved_bytes_peak_delta": 22666018816, + "segment_count_current": 436, + "segment_count_current_delta": 58, + "segment_count_peak": 436, + "segment_count_peak_delta": 58, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T131424784358Z.pickle", + "timestamp_utc": "2026-08-01T13:14:24.893756Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T131424784358Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T131424784358Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c2c83dffbb66daaa669fe5d6d1ad4de9df0d0a30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T131424784358Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae7b262b2389f5def72ea15f02de26c14d08e2780782ae1d6cf20d5b1068e8e8 +size 6070095 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T131426102026Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T131426102026Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5fb02bfba590f66de802b46488c7fa56fbc5f902 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T131426102026Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116761419776, + "active_bytes_peak_delta": 369632256, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116761419776, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115704455168, + "device": 1, + "inactive_split_bytes_current": 11474928640, + "inactive_split_bytes_peak": 17136631808, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 121597067264, + "reserved_bytes_peak": 121597067264, + "segment_count_current": 436, + "segment_count_peak": 436, + "timestamp_utc": "2026-08-01T13:14:26.101616Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115704455168, + "allocated_bytes_peak_delta": 6674432, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116391787520, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115697780736, + "device": 1, + "inactive_split_bytes_current": 9595880448, + "inactive_split_bytes_peak": 17136631808, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 121597067264, + "reserved_bytes_peak": 121597067264, + "segment_count_current": 436, + "segment_count_peak": 436, + "timestamp_utc": "2026-08-01T13:14:24.908509Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11474928640, + "inactive_split_bytes_current_delta": 1879048192, + "inactive_split_bytes_peak": 17136631808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 121597067264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121597067264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 436, + "segment_count_current_delta": 0, + "segment_count_peak": 436, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T131426102026Z.pickle", + "timestamp_utc": "2026-08-01T13:14:26.210572Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T131426102026Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T131426102026Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cefa918a66fc6d5cea73dec82fc54bc70d6cc8e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T131426102026Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40c3bba57faa000882e3c77aef8856b538f578dc0053a791a3ebee1ef22a3748 +size 6589401 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T131427847975Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T131427847975Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b7b7e389169c624ace89b91c3a07cfe2993a273 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T131427847975Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 160481178624, + "active_bytes_current_delta": 87973558272, + "active_bytes_peak": 160543044608, + "active_bytes_peak_delta": 43781624832, + "after": { + "active_bytes_current": 160481178624, + "active_bytes_peak": 160543044608, + "allocated_bytes_current": 104982385664, + "allocated_bytes_peak": 115704455168, + "device": 1, + "inactive_split_bytes_current": 14089948160, + "inactive_split_bytes_peak": 21201323008, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 182856974336, + "reserved_bytes_peak": 182856974336, + "segment_count_current": 664, + "segment_count_peak": 664, + "timestamp_utc": "2026-08-01T13:14:27.847311Z", + "world_size": 8 + }, + "allocated_bytes_current": 104982385664, + "allocated_bytes_current_delta": 32474765312, + "allocated_bytes_peak": 115704455168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116761419776, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115704455168, + "device": 1, + "inactive_split_bytes_current": 11474928640, + "inactive_split_bytes_peak": 17136631808, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 121597067264, + "reserved_bytes_peak": 121597067264, + "segment_count_current": 436, + "segment_count_peak": 436, + "timestamp_utc": "2026-08-01T13:14:26.225277Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 14089948160, + "inactive_split_bytes_current_delta": 2615019520, + "inactive_split_bytes_peak": 21201323008, + "inactive_split_bytes_peak_delta": 4064691200, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 182856974336, + "reserved_bytes_current_delta": 61259907072, + "reserved_bytes_peak": 182856974336, + "reserved_bytes_peak_delta": 61259907072, + "segment_count_current": 664, + "segment_count_current_delta": 228, + "segment_count_peak": 664, + "segment_count_peak_delta": 228, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T131427847975Z.pickle", + "timestamp_utc": "2026-08-01T13:14:27.952743Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T131427847975Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T131427847975Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f4ed7dd44058ee1f2d05f02d7faf1319a46a754 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T131427847975Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90373ca87b842cf2782e86763d7e54e74f7ee3372346422abfb7966124fd3046 +size 5261278 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T131423399498Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T131423399498Z.json new file mode 100644 index 0000000000000000000000000000000000000000..68d09409fbc54c40688c2f8a720f5779336d52b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T131423399498Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 92749819392, + "active_bytes_peak_delta": 57283483136, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92749819392, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92061167616, + "device": 2, + "inactive_split_bytes_current": 6947177472, + "inactive_split_bytes_peak": 12396578304, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 99105112064, + "reserved_bytes_peak": 99105112064, + "segment_count_current": 381, + "segment_count_peak": 381, + "timestamp_utc": "2026-08-01T13:14:23.398906Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 92061167616, + "allocated_bytes_peak_delta": 56594831360, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:13:14.705987Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6947177472, + "inactive_split_bytes_current_delta": 6814358528, + "inactive_split_bytes_peak": 12396578304, + "inactive_split_bytes_peak_delta": 12236192256, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 99105112064, + "reserved_bytes_current_delta": 59861106688, + "reserved_bytes_peak": 99105112064, + "reserved_bytes_peak_delta": 59861106688, + "segment_count_current": 381, + "segment_count_current_delta": 310, + "segment_count_peak": 381, + "segment_count_peak_delta": 310, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T131423399498Z.pickle", + "timestamp_utc": "2026-08-01T13:14:23.515091Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T131423399498Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T131423399498Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..720c9b0165a1bcf750ac668524970ff7fa8cd4dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T131423399498Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e40a1ea9c2743c5fc8ae2a9316e74bde7f5fcd0477d5c8c3554182590d510bd +size 5943909 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T131424784367Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T131424784367Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d488d491c7fc76a944802ea1d21c9d810378a29 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T131424784367Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116386227200, + "active_bytes_peak_delta": 23636407808, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116386227200, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115694444544, + "device": 2, + "inactive_split_bytes_current": 8650064896, + "inactive_split_bytes_peak": 15885831168, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 122341556224, + "reserved_bytes_peak": 122341556224, + "segment_count_current": 446, + "segment_count_peak": 446, + "timestamp_utc": "2026-08-01T13:14:24.783966Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115694444544, + "allocated_bytes_peak_delta": 23633276928, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92749819392, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92061167616, + "device": 2, + "inactive_split_bytes_current": 6947177472, + "inactive_split_bytes_peak": 12396578304, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 99105112064, + "reserved_bytes_peak": 99105112064, + "segment_count_current": 381, + "segment_count_peak": 381, + "timestamp_utc": "2026-08-01T13:14:23.533620Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8650064896, + "inactive_split_bytes_current_delta": 1702887424, + "inactive_split_bytes_peak": 15885831168, + "inactive_split_bytes_peak_delta": 3489252864, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 122341556224, + "reserved_bytes_current_delta": 23236444160, + "reserved_bytes_peak": 122341556224, + "reserved_bytes_peak_delta": 23236444160, + "segment_count_current": 446, + "segment_count_current_delta": 65, + "segment_count_peak": 446, + "segment_count_peak_delta": 65, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T131424784367Z.pickle", + "timestamp_utc": "2026-08-01T13:14:24.889767Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T131424784367Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T131424784367Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..319926f0bd42c241d9b258de890fb3bf8c0c7bc5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T131424784367Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6bfc1be2321953e7dee6d7110f9b8cedf00a04d8896a9053d8a26e8110ec97a +size 6069133 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T131426102068Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T131426102068Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61c43f505ebb5a7773ca26c665cb5e7b8537087f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T131426102068Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116763852800, + "active_bytes_peak_delta": 377625600, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116763852800, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115706888192, + "device": 2, + "inactive_split_bytes_current": 8358560768, + "inactive_split_bytes_peak": 16405276672, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 122341556224, + "reserved_bytes_peak": 122341556224, + "segment_count_current": 446, + "segment_count_peak": 446, + "timestamp_utc": "2026-08-01T13:14:26.101664Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115706888192, + "allocated_bytes_peak_delta": 12443648, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116386227200, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115694444544, + "device": 2, + "inactive_split_bytes_current": 8650064896, + "inactive_split_bytes_peak": 15885831168, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 122341556224, + "reserved_bytes_peak": 122341556224, + "segment_count_current": 446, + "segment_count_peak": 446, + "timestamp_utc": "2026-08-01T13:14:24.902312Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8358560768, + "inactive_split_bytes_current_delta": -291504128, + "inactive_split_bytes_peak": 16405276672, + "inactive_split_bytes_peak_delta": 519445504, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 122341556224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122341556224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 446, + "segment_count_current_delta": 0, + "segment_count_peak": 446, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T131426102068Z.pickle", + "timestamp_utc": "2026-08-01T13:14:26.207743Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T131426102068Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T131426102068Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0cc477b9d6199c62f87b3deaaeffd2236d98ebae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T131426102068Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e168bee3b711a834d1b9f4e1fd2760b284518429257f3f5b01eadbe7c7b61ae4 +size 6590410 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T131427713077Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T131427713077Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dfd2d0e10287a9d66da2c58aa3d4276481b69704 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T131427713077Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 164959241216, + "active_bytes_current_delta": 92451620864, + "active_bytes_peak": 165021369344, + "active_bytes_peak_delta": 48257516544, + "after": { + "active_bytes_current": 164959241216, + "active_bytes_peak": 165021369344, + "allocated_bytes_current": 108755805184, + "allocated_bytes_peak": 115706888192, + "device": 2, + "inactive_split_bytes_current": 12803750912, + "inactive_split_bytes_peak": 19631080960, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 182869557248, + "reserved_bytes_peak": 182869557248, + "segment_count_current": 664, + "segment_count_peak": 664, + "timestamp_utc": "2026-08-01T13:14:27.712452Z", + "world_size": 8 + }, + "allocated_bytes_current": 108755805184, + "allocated_bytes_current_delta": 36248184832, + "allocated_bytes_peak": 115706888192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116763852800, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115706888192, + "device": 2, + "inactive_split_bytes_current": 8358560768, + "inactive_split_bytes_peak": 16405276672, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 122341556224, + "reserved_bytes_peak": 122341556224, + "segment_count_current": 446, + "segment_count_peak": 446, + "timestamp_utc": "2026-08-01T13:14:26.219837Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12803750912, + "inactive_split_bytes_current_delta": 4445190144, + "inactive_split_bytes_peak": 19631080960, + "inactive_split_bytes_peak_delta": 3225804288, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 182869557248, + "reserved_bytes_current_delta": 60528001024, + "reserved_bytes_peak": 182869557248, + "reserved_bytes_peak_delta": 60528001024, + "segment_count_current": 664, + "segment_count_current_delta": 218, + "segment_count_peak": 664, + "segment_count_peak_delta": 218, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T131427713077Z.pickle", + "timestamp_utc": "2026-08-01T13:14:27.817034Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T131427713077Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T131427713077Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5799de4b8eec394875c36c097e3f7cbd528853ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T131427713077Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ef634a823b3ee7e4c9a03c6b952d77c9cc7ccdc69ab4271df481959faf4354f +size 5399760 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T131423399542Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T131423399542Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d2cafe6e68c35475018891205fae806401920e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T131423399542Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 92406411264, + "active_bytes_peak_delta": 56940075008, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92406411264, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92054089728, + "device": 3, + "inactive_split_bytes_current": 6418695168, + "inactive_split_bytes_peak": 11933894144, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 98050244608, + "reserved_bytes_peak": 98050244608, + "segment_count_current": 378, + "segment_count_peak": 378, + "timestamp_utc": "2026-08-01T13:14:23.398897Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 92054089728, + "allocated_bytes_peak_delta": 56587753472, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:13:14.705817Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6418695168, + "inactive_split_bytes_current_delta": 6285876224, + "inactive_split_bytes_peak": 11933894144, + "inactive_split_bytes_peak_delta": 11773508096, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 98050244608, + "reserved_bytes_current_delta": 58806239232, + "reserved_bytes_peak": 98050244608, + "reserved_bytes_peak_delta": 58806239232, + "segment_count_current": 378, + "segment_count_current_delta": 307, + "segment_count_peak": 378, + "segment_count_peak_delta": 307, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T131423399542Z.pickle", + "timestamp_utc": "2026-08-01T13:14:23.519132Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T131423399542Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T131423399542Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80f0bd9ff94f3a5961c2b582b4789ba41e39f0dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T131423399542Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5132232a59e75588ca63ec1c0e5abdccf0fac1fa7f5d1a23270af97bd74e94b9 +size 5943327 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T131424784367Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T131424784367Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1841ba9f740d48acaaef271cbf2aa71409e906d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T131424784367Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116402725888, + "active_bytes_peak_delta": 23996314624, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116402725888, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115701542912, + "device": 3, + "inactive_split_bytes_current": 8541012992, + "inactive_split_bytes_peak": 12766619648, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 121171345408, + "reserved_bytes_peak": 121171345408, + "segment_count_current": 439, + "segment_count_peak": 439, + "timestamp_utc": "2026-08-01T13:14:24.783951Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115701542912, + "allocated_bytes_peak_delta": 23647453184, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92406411264, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92054089728, + "device": 3, + "inactive_split_bytes_current": 6418695168, + "inactive_split_bytes_peak": 11933894144, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 98050244608, + "reserved_bytes_peak": 98050244608, + "segment_count_current": 378, + "segment_count_peak": 378, + "timestamp_utc": "2026-08-01T13:14:23.540354Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8541012992, + "inactive_split_bytes_current_delta": 2122317824, + "inactive_split_bytes_peak": 12766619648, + "inactive_split_bytes_peak_delta": 832725504, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 121171345408, + "reserved_bytes_current_delta": 23121100800, + "reserved_bytes_peak": 121171345408, + "reserved_bytes_peak_delta": 23121100800, + "segment_count_current": 439, + "segment_count_current_delta": 61, + "segment_count_peak": 439, + "segment_count_peak_delta": 61, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T131424784367Z.pickle", + "timestamp_utc": "2026-08-01T13:14:24.894224Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T131424784367Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T131424784367Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..88fc2804933736ce2bd80b8f73c7873c7c3c3beb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T131424784367Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9279f94d4923c2f6d28c09c8f35d465dd9dda1397c4bfee71d3417daf05ad7b0 +size 6073254 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T131426102011Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T131426102011Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42ea1b910ae2b565a779ea8dd018960033d9502c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T131426102011Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116765274112, + "active_bytes_peak_delta": 362548224, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116765274112, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115708379136, + "device": 3, + "inactive_split_bytes_current": 9946104832, + "inactive_split_bytes_peak": 16292065280, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 121995526144, + "reserved_bytes_peak": 121995526144, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T13:14:26.101595Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115708379136, + "allocated_bytes_peak_delta": 6836224, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116402725888, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115701542912, + "device": 3, + "inactive_split_bytes_current": 8541012992, + "inactive_split_bytes_peak": 12766619648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 121171345408, + "reserved_bytes_peak": 121171345408, + "segment_count_current": 439, + "segment_count_peak": 439, + "timestamp_utc": "2026-08-01T13:14:24.910608Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9946104832, + "inactive_split_bytes_current_delta": 1405091840, + "inactive_split_bytes_peak": 16292065280, + "inactive_split_bytes_peak_delta": 3525445632, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 121995526144, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 121995526144, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 440, + "segment_count_current_delta": 1, + "segment_count_peak": 440, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T131426102011Z.pickle", + "timestamp_utc": "2026-08-01T13:14:26.212798Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T131426102011Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T131426102011Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a8f372e79d803d54a8361aa0adfd60e1b95fc4e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T131426102011Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92c9e5c6b76c10e80c4e14f12be398fccf55b1c2742643328574510e24097896 +size 6067017 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T131427778888Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T131427778888Z.json new file mode 100644 index 0000000000000000000000000000000000000000..097d07bf5f99728ecd2eb039633741241d3daacf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T131427778888Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 162072452096, + "active_bytes_current_delta": 89564831744, + "active_bytes_peak": 162072452096, + "active_bytes_peak_delta": 45307177984, + "after": { + "active_bytes_current": 162072452096, + "active_bytes_peak": 162072452096, + "allocated_bytes_current": 106221337600, + "allocated_bytes_peak": 115708379136, + "device": 3, + "inactive_split_bytes_current": 13668885504, + "inactive_split_bytes_peak": 20723958784, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 182519332864, + "reserved_bytes_peak": 182519332864, + "segment_count_current": 659, + "segment_count_peak": 659, + "timestamp_utc": "2026-08-01T13:14:27.778243Z", + "world_size": 8 + }, + "allocated_bytes_current": 106221337600, + "allocated_bytes_current_delta": 33713717248, + "allocated_bytes_peak": 115708379136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116765274112, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115708379136, + "device": 3, + "inactive_split_bytes_current": 9946104832, + "inactive_split_bytes_peak": 16292065280, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 121995526144, + "reserved_bytes_peak": 121995526144, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T13:14:26.228965Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 13668885504, + "inactive_split_bytes_current_delta": 3722780672, + "inactive_split_bytes_peak": 20723958784, + "inactive_split_bytes_peak_delta": 4431893504, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 182519332864, + "reserved_bytes_current_delta": 60523806720, + "reserved_bytes_peak": 182519332864, + "reserved_bytes_peak_delta": 60523806720, + "segment_count_current": 659, + "segment_count_current_delta": 219, + "segment_count_peak": 659, + "segment_count_peak_delta": 219, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T131427778888Z.pickle", + "timestamp_utc": "2026-08-01T13:14:27.880825Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T131427778888Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T131427778888Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6061665a4369a5673b621c83cb28002070c0e653 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T131427778888Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8cacd2859e1bf068ba8e1e4a3693a9ec8d8eeff1fbb081e262df73daf19aded +size 5324160 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T131423399706Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T131423399706Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ce127e748544219a645f4184778df449931e53d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T131423399706Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509401088, + "active_bytes_current_delta": 37277503488, + "active_bytes_peak": 92403302400, + "active_bytes_peak_delta": 56936523776, + "after": { + "active_bytes_current": 72509401088, + "active_bytes_peak": 92403302400, + "allocated_bytes_current": 72509401088, + "allocated_bytes_peak": 92050980864, + "device": 4, + "inactive_split_bytes_current": 5242509312, + "inactive_split_bytes_peak": 11088975360, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 97167343616, + "reserved_bytes_peak": 97167343616, + "segment_count_current": 377, + "segment_count_peak": 377, + "timestamp_utc": "2026-08-01T13:14:23.399027Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509401088, + "allocated_bytes_current_delta": 37277503488, + "allocated_bytes_peak": 92050980864, + "allocated_bytes_peak_delta": 56584202240, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:13:14.706444Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5242509312, + "inactive_split_bytes_current_delta": 5110132736, + "inactive_split_bytes_peak": 11088975360, + "inactive_split_bytes_peak_delta": 10935565824, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 97167343616, + "reserved_bytes_current_delta": 57923338240, + "reserved_bytes_peak": 97167343616, + "reserved_bytes_peak_delta": 57923338240, + "segment_count_current": 377, + "segment_count_current_delta": 306, + "segment_count_peak": 377, + "segment_count_peak_delta": 306, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T131423399706Z.pickle", + "timestamp_utc": "2026-08-01T13:14:23.524364Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T131423399706Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T131423399706Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f554c8b181ebe44e9da33c28d021edf1b328678c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T131423399706Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56cf9f9431654505f52c457a2d961ba8168e23f3e1566cd156cef672299b57e5 +size 5951885 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T131424784700Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T131424784700Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ae51df319ed4b0b80878e776b051f469c02ea48 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T131424784700Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509401088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116038368256, + "active_bytes_peak_delta": 23635065856, + "after": { + "active_bytes_current": 72509401088, + "active_bytes_peak": 116038368256, + "allocated_bytes_current": 72509401088, + "allocated_bytes_peak": 115700074496, + "device": 4, + "inactive_split_bytes_current": 9594099712, + "inactive_split_bytes_peak": 13907863552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121949388800, + "reserved_bytes_peak": 121949388800, + "segment_count_current": 438, + "segment_count_peak": 438, + "timestamp_utc": "2026-08-01T13:14:24.784120Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509401088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115700074496, + "allocated_bytes_peak_delta": 23649093632, + "before": { + "active_bytes_current": 72509401088, + "active_bytes_peak": 92403302400, + "allocated_bytes_current": 72509401088, + "allocated_bytes_peak": 92050980864, + "device": 4, + "inactive_split_bytes_current": 5242509312, + "inactive_split_bytes_peak": 11088975360, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 97167343616, + "reserved_bytes_peak": 97167343616, + "segment_count_current": 377, + "segment_count_peak": 377, + "timestamp_utc": "2026-08-01T13:14:23.544391Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9594099712, + "inactive_split_bytes_current_delta": 4351590400, + "inactive_split_bytes_peak": 13907863552, + "inactive_split_bytes_peak_delta": 2818888192, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121949388800, + "reserved_bytes_current_delta": 24782045184, + "reserved_bytes_peak": 121949388800, + "reserved_bytes_peak_delta": 24782045184, + "segment_count_current": 438, + "segment_count_current_delta": 61, + "segment_count_peak": 438, + "segment_count_peak_delta": 61, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T131424784700Z.pickle", + "timestamp_utc": "2026-08-01T13:14:24.907109Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T131424784700Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T131424784700Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a73546be5663784db17d2ae674afc25b2aeff53 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T131424784700Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b331a631a0bbf33a157c2abdb74835f80c57559328da529df6887387489bb0b +size 6078214 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T131426102276Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T131426102276Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c37d67199bf721c2228020498d0830aff4aa5be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T131426102276Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72509401088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116410091520, + "active_bytes_peak_delta": 371723264, + "after": { + "active_bytes_current": 72509401088, + "active_bytes_peak": 116410091520, + "allocated_bytes_current": 72509401088, + "allocated_bytes_peak": 115705448448, + "device": 4, + "inactive_split_bytes_current": 8419694592, + "inactive_split_bytes_peak": 15782512640, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121949388800, + "reserved_bytes_peak": 121949388800, + "segment_count_current": 438, + "segment_count_peak": 438, + "timestamp_utc": "2026-08-01T13:14:26.101757Z", + "world_size": 8 + }, + "allocated_bytes_current": 72509401088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115705448448, + "allocated_bytes_peak_delta": 5373952, + "before": { + "active_bytes_current": 72509401088, + "active_bytes_peak": 116038368256, + "allocated_bytes_current": 72509401088, + "allocated_bytes_peak": 115700074496, + "device": 4, + "inactive_split_bytes_current": 9594099712, + "inactive_split_bytes_peak": 13907863552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121949388800, + "reserved_bytes_peak": 121949388800, + "segment_count_current": 438, + "segment_count_peak": 438, + "timestamp_utc": "2026-08-01T13:14:24.925029Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8419694592, + "inactive_split_bytes_current_delta": -1174405120, + "inactive_split_bytes_peak": 15782512640, + "inactive_split_bytes_peak_delta": 1874649088, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121949388800, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121949388800, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 438, + "segment_count_current_delta": 0, + "segment_count_peak": 438, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T131426102276Z.pickle", + "timestamp_utc": "2026-08-01T13:14:26.221681Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T131426102276Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T131426102276Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9cb2eda0c8ddc545c8bd9b29f98b3aafc21018fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T131426102276Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa8ef31a783f410c2982eed5d28135efb132d47592a6601269389276e81b9ec5 +size 6596865 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T131427806284Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T131427806284Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5a1e690b41a521ace37eeb4d3bb41ca29387f32 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T131427806284Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 164318900224, + "active_bytes_current_delta": 91809499136, + "active_bytes_peak": 164318900224, + "active_bytes_peak_delta": 47908808704, + "after": { + "active_bytes_current": 164318900224, + "active_bytes_peak": 164318900224, + "allocated_bytes_current": 108114415616, + "allocated_bytes_peak": 115705448448, + "device": 4, + "inactive_split_bytes_current": 12466819072, + "inactive_split_bytes_peak": 19023223296, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 182917791744, + "reserved_bytes_peak": 182917791744, + "segment_count_current": 668, + "segment_count_peak": 668, + "timestamp_utc": "2026-08-01T13:14:27.805662Z", + "world_size": 8 + }, + "allocated_bytes_current": 108114415616, + "allocated_bytes_current_delta": 35605014528, + "allocated_bytes_peak": 115705448448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72509401088, + "active_bytes_peak": 116410091520, + "allocated_bytes_current": 72509401088, + "allocated_bytes_peak": 115705448448, + "device": 4, + "inactive_split_bytes_current": 8419694592, + "inactive_split_bytes_peak": 15782512640, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121949388800, + "reserved_bytes_peak": 121949388800, + "segment_count_current": 438, + "segment_count_peak": 438, + "timestamp_utc": "2026-08-01T13:14:26.239433Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 12466819072, + "inactive_split_bytes_current_delta": 4047124480, + "inactive_split_bytes_peak": 19023223296, + "inactive_split_bytes_peak_delta": 3240710656, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 182917791744, + "reserved_bytes_current_delta": 60968402944, + "reserved_bytes_peak": 182917791744, + "reserved_bytes_peak_delta": 60968402944, + "segment_count_current": 668, + "segment_count_current_delta": 230, + "segment_count_peak": 668, + "segment_count_peak_delta": 230, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T131427806284Z.pickle", + "timestamp_utc": "2026-08-01T13:14:27.913606Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T131427806284Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T131427806284Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e57fd249ac1b6289f3c18765bc3c1fcbbd2c4d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T131427806284Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bff7a86291856a9956cb86d20389ba2c3f540dd2964712d20cbb177a5773f1d +size 5395241 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T131423399534Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T131423399534Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fa1335091768fa87797fdf88818cd1a0c61b3cfe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T131423399534Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 92415870976, + "active_bytes_peak_delta": 56949534720, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92415870976, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92063549440, + "device": 5, + "inactive_split_bytes_current": 6540329984, + "inactive_split_bytes_peak": 12061180416, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 97991524352, + "reserved_bytes_peak": 97991524352, + "segment_count_current": 376, + "segment_count_peak": 376, + "timestamp_utc": "2026-08-01T13:14:23.398932Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 92063549440, + "allocated_bytes_peak_delta": 56597213184, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:13:14.705843Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6540329984, + "inactive_split_bytes_current_delta": 6407511040, + "inactive_split_bytes_peak": 12061180416, + "inactive_split_bytes_peak_delta": 11900794368, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 97991524352, + "reserved_bytes_current_delta": 58747518976, + "reserved_bytes_peak": 97991524352, + "reserved_bytes_peak_delta": 58747518976, + "segment_count_current": 376, + "segment_count_current_delta": 305, + "segment_count_peak": 376, + "segment_count_peak_delta": 305, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T131423399534Z.pickle", + "timestamp_utc": "2026-08-01T13:14:23.520571Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T131423399534Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T131423399534Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..883260b16575fc7b5434e77c127ac4a28be67e1f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T131423399534Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1358bb2c2f3d8b8444e5854981f3860a97d12625516afced7b09e21cabc66b2d +size 5943197 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T131424784379Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T131424784379Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06b3bdca530491a8f8336cb307999ba6ba875d4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T131424784379Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116394894336, + "active_bytes_peak_delta": 23979023360, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116394894336, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115690251264, + "device": 5, + "inactive_split_bytes_current": 9417622528, + "inactive_split_bytes_peak": 16475581440, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 122284933120, + "reserved_bytes_peak": 122284933120, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T13:14:24.783967Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115690251264, + "allocated_bytes_peak_delta": 23626701824, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92415870976, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92063549440, + "device": 5, + "inactive_split_bytes_current": 6540329984, + "inactive_split_bytes_peak": 12061180416, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 97991524352, + "reserved_bytes_peak": 97991524352, + "segment_count_current": 376, + "segment_count_peak": 376, + "timestamp_utc": "2026-08-01T13:14:23.537652Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9417622528, + "inactive_split_bytes_current_delta": 2877292544, + "inactive_split_bytes_peak": 16475581440, + "inactive_split_bytes_peak_delta": 4414401024, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 122284933120, + "reserved_bytes_current_delta": 24293408768, + "reserved_bytes_peak": 122284933120, + "reserved_bytes_peak_delta": 24293408768, + "segment_count_current": 440, + "segment_count_current_delta": 64, + "segment_count_peak": 440, + "segment_count_peak_delta": 64, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T131424784379Z.pickle", + "timestamp_utc": "2026-08-01T13:14:24.893185Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T131424784379Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T131424784379Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..86b243ead52c38222e6eba84037fc6aa8382eced --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T131424784379Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:669204aaa56a4ded7e8f312ced2f26035d30b823260e34af80a71bc10d94782a +size 6070734 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T131426102024Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T131426102024Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a257ead05e0a896ff2a60dfd38c43f85589bf076 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T131426102024Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116763779072, + "active_bytes_peak_delta": 368884736, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116763779072, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115706814464, + "device": 5, + "inactive_split_bytes_current": 10417964032, + "inactive_split_bytes_peak": 16641650688, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 122284933120, + "reserved_bytes_peak": 122284933120, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T13:14:26.101625Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115706814464, + "allocated_bytes_peak_delta": 16563200, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116394894336, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115690251264, + "device": 5, + "inactive_split_bytes_current": 9417622528, + "inactive_split_bytes_peak": 16475581440, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 122284933120, + "reserved_bytes_peak": 122284933120, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T13:14:24.907089Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10417964032, + "inactive_split_bytes_current_delta": 1000341504, + "inactive_split_bytes_peak": 16641650688, + "inactive_split_bytes_peak_delta": 166069248, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 122284933120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122284933120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 440, + "segment_count_current_delta": 0, + "segment_count_peak": 440, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T131426102024Z.pickle", + "timestamp_utc": "2026-08-01T13:14:26.214054Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T131426102024Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T131426102024Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e3cc8d91cc7b97b1f57d1ebd4a439b014df26d8b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T131426102024Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52aed03673ee06e8a392fb2ce6d8f4583b4b35f7949836e32caec44060f5067a +size 6589949 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T131427672767Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T131427672767Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4906a6742864107230311fef9acdca9738ffdfdf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T131427672767Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 161261759488, + "active_bytes_current_delta": 88754139136, + "active_bytes_peak": 161548628992, + "active_bytes_peak_delta": 44784849920, + "after": { + "active_bytes_current": 161261759488, + "active_bytes_peak": 161548628992, + "allocated_bytes_current": 105409596416, + "allocated_bytes_peak": 115706814464, + "device": 5, + "inactive_split_bytes_current": 14253085696, + "inactive_split_bytes_peak": 20688307712, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 182859071488, + "reserved_bytes_peak": 182859071488, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T13:14:27.672167Z", + "world_size": 8 + }, + "allocated_bytes_current": 105409596416, + "allocated_bytes_current_delta": 32901976064, + "allocated_bytes_peak": 115706814464, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116763779072, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115706814464, + "device": 5, + "inactive_split_bytes_current": 10417964032, + "inactive_split_bytes_peak": 16641650688, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 122284933120, + "reserved_bytes_peak": 122284933120, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T13:14:26.227832Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 14253085696, + "inactive_split_bytes_current_delta": 3835121664, + "inactive_split_bytes_peak": 20688307712, + "inactive_split_bytes_peak_delta": 4046657024, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 182859071488, + "reserved_bytes_current_delta": 60574138368, + "reserved_bytes_peak": 182859071488, + "reserved_bytes_peak_delta": 60574138368, + "segment_count_current": 665, + "segment_count_current_delta": 225, + "segment_count_peak": 665, + "segment_count_peak_delta": 225, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T131427672767Z.pickle", + "timestamp_utc": "2026-08-01T13:14:27.772524Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T131427672767Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T131427672767Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d18ff88d21624185882036c270a5a91e6064dccf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T131427672767Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0263fc98eef1b134d83e7bd4bf39635c9e8ecf8eda015bf97c8094893416da26 +size 5343129 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T131423399501Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T131423399501Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4963b3e2197cf8ed17ce201484df7050679ff6a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T131423399501Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 37276165120, + "active_bytes_peak": 92400381952, + "active_bytes_peak_delta": 56934045696, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92400381952, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92048060416, + "device": 6, + "inactive_split_bytes_current": 6596953088, + "inactive_split_bytes_peak": 12994336256, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98461286400, + "reserved_bytes_peak": 98461286400, + "segment_count_current": 379, + "segment_count_peak": 379, + "timestamp_utc": "2026-08-01T13:14:23.398879Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 37276165120, + "allocated_bytes_peak": 92048060416, + "allocated_bytes_peak_delta": 56581724160, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:13:14.706045Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6596953088, + "inactive_split_bytes_current_delta": 6464134144, + "inactive_split_bytes_peak": 12994336256, + "inactive_split_bytes_peak_delta": 12833950208, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98461286400, + "reserved_bytes_current_delta": 59217281024, + "reserved_bytes_peak": 98461286400, + "reserved_bytes_peak_delta": 59217281024, + "segment_count_current": 379, + "segment_count_current_delta": 308, + "segment_count_peak": 379, + "segment_count_peak_delta": 308, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T131423399501Z.pickle", + "timestamp_utc": "2026-08-01T13:14:23.519490Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T131423399501Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T131423399501Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fbfc0fd93a6782327db98d5ae6ad8ec1e68d17d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T131423399501Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef15f8ac949295c0b8c2a9cca69bfdf54cb92e25d53f0efb578672c8924d1885 +size 5943634 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T131424784359Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T131424784359Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c0f8501a3a0b038987fe96ccf835bc24a9ce1fdd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T131424784359Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116394867712, + "active_bytes_peak_delta": 23994485760, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116394867712, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115699767296, + "device": 6, + "inactive_split_bytes_current": 9417622528, + "inactive_split_bytes_peak": 14832564224, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121697730560, + "reserved_bytes_peak": 121697730560, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T13:14:24.783936Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115699767296, + "allocated_bytes_peak_delta": 23651706880, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 92400381952, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 92048060416, + "device": 6, + "inactive_split_bytes_current": 6596953088, + "inactive_split_bytes_peak": 12994336256, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98461286400, + "reserved_bytes_peak": 98461286400, + "segment_count_current": 379, + "segment_count_peak": 379, + "timestamp_utc": "2026-08-01T13:14:23.540397Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9417622528, + "inactive_split_bytes_current_delta": 2820669440, + "inactive_split_bytes_peak": 14832564224, + "inactive_split_bytes_peak_delta": 1838227968, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121697730560, + "reserved_bytes_current_delta": 23236444160, + "reserved_bytes_peak": 121697730560, + "reserved_bytes_peak_delta": 23236444160, + "segment_count_current": 442, + "segment_count_current_delta": 63, + "segment_count_peak": 442, + "segment_count_peak_delta": 63, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T131424784359Z.pickle", + "timestamp_utc": "2026-08-01T13:14:24.896357Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T131424784359Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T131424784359Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..24a79c49893e979e6584c15a39ee0279456cb825 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T131424784359Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8703c61d185cedebe3c72d1e1d70c3fddb5c62a67738bd43dfc0341dbca85ad +size 6056424 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T131426102074Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T131426102074Z.json new file mode 100644 index 0000000000000000000000000000000000000000..df734f2867f843a9342f8599e7d3e38b4799b717 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T131426102074Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620352, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116764938240, + "active_bytes_peak_delta": 370070528, + "after": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116764938240, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115707973632, + "device": 6, + "inactive_split_bytes_current": 8421475328, + "inactive_split_bytes_peak": 15352283136, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121697730560, + "reserved_bytes_peak": 121697730560, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T13:14:26.101653Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620352, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115707973632, + "allocated_bytes_peak_delta": 8206336, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116394867712, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115699767296, + "device": 6, + "inactive_split_bytes_current": 9417622528, + "inactive_split_bytes_peak": 14832564224, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121697730560, + "reserved_bytes_peak": 121697730560, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T13:14:24.912455Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8421475328, + "inactive_split_bytes_current_delta": -996147200, + "inactive_split_bytes_peak": 15352283136, + "inactive_split_bytes_peak_delta": 519718912, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121697730560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121697730560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 442, + "segment_count_current_delta": 0, + "segment_count_peak": 442, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T131426102074Z.pickle", + "timestamp_utc": "2026-08-01T13:14:26.214170Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T131426102074Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T131426102074Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94e9fa1f8374a1b01ba56b9e5c11c0c1a69f1302 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T131426102074Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f51613b975d8cfc431a8b4892b19ae69ce9547207da261ffde00e6145a8569e +size 6589667 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T131427749458Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T131427749458Z.json new file mode 100644 index 0000000000000000000000000000000000000000..41408b7b024456a91c010be413f6cee6b7855430 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T131427749458Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 164320396288, + "active_bytes_current_delta": 91812775936, + "active_bytes_peak": 164320396288, + "active_bytes_peak_delta": 47555458048, + "after": { + "active_bytes_current": 164320396288, + "active_bytes_peak": 164320396288, + "allocated_bytes_current": 108115911680, + "allocated_bytes_peak": 115707973632, + "device": 6, + "inactive_split_bytes_current": 12337396736, + "inactive_split_bytes_peak": 18915296768, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 182915694592, + "reserved_bytes_peak": 182915694592, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T13:14:27.748864Z", + "world_size": 8 + }, + "allocated_bytes_current": 108115911680, + "allocated_bytes_current_delta": 35608291328, + "allocated_bytes_peak": 115707973632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620352, + "active_bytes_peak": 116764938240, + "allocated_bytes_current": 72507620352, + "allocated_bytes_peak": 115707973632, + "device": 6, + "inactive_split_bytes_current": 8421475328, + "inactive_split_bytes_peak": 15352283136, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121697730560, + "reserved_bytes_peak": 121697730560, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T13:14:26.229871Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 12337396736, + "inactive_split_bytes_current_delta": 3915921408, + "inactive_split_bytes_peak": 18915296768, + "inactive_split_bytes_peak_delta": 3563013632, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 182915694592, + "reserved_bytes_current_delta": 61217964032, + "reserved_bytes_peak": 182915694592, + "reserved_bytes_peak_delta": 61217964032, + "segment_count_current": 665, + "segment_count_current_delta": 223, + "segment_count_peak": 665, + "segment_count_peak_delta": 223, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T131427749458Z.pickle", + "timestamp_utc": "2026-08-01T13:14:27.854722Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T131427749458Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T131427749458Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe46e4eb3e19dbfa1295753ef9642c28b3ddf7f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T131427749458Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bdfaf9ae900cd4724c049c72f0dc557e94e7325e0d4b5e1bf486ef9e5fe4319 +size 5387489 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T131423399471Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T131423399471Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b60816a6aab0626c6064744ed841a85a71c64a88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T131423399471Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 37276165632, + "active_bytes_peak": 92755963904, + "active_bytes_peak_delta": 57289627648, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 92755963904, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 92051439616, + "device": 7, + "inactive_split_bytes_current": 8182399488, + "inactive_split_bytes_peak": 11172406272, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 99870572544, + "reserved_bytes_peak": 99870572544, + "segment_count_current": 380, + "segment_count_peak": 380, + "timestamp_utc": "2026-08-01T13:14:23.398879Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 37276165632, + "allocated_bytes_peak": 92051439616, + "allocated_bytes_peak_delta": 56585103360, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:13:14.709216Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8182399488, + "inactive_split_bytes_current_delta": 8049580544, + "inactive_split_bytes_peak": 11172406272, + "inactive_split_bytes_peak_delta": 11012020224, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 99870572544, + "reserved_bytes_current_delta": 60626567168, + "reserved_bytes_peak": 99870572544, + "reserved_bytes_peak_delta": 60626567168, + "segment_count_current": 380, + "segment_count_current_delta": 309, + "segment_count_peak": 380, + "segment_count_peak_delta": 309, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T131423399471Z.pickle", + "timestamp_utc": "2026-08-01T13:14:23.521063Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T131423399471Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T131423399471Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d238711ab6d8e812ec4b88e1f58465a939797104 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T131423399471Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8608df2f23eb916a8468c74e11411b21b1126e99f0a08b8a9134f9252dc1ea24 +size 5945449 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T131424784485Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T131424784485Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ff38deb6fb493d3bb0b93c7f5619fbb2d27758da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T131424784485Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116399812096, + "active_bytes_peak_delta": 23643848192, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 116399812096, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 115695169024, + "device": 7, + "inactive_split_bytes_current": 9769943552, + "inactive_split_bytes_peak": 15294089728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 121993428992, + "reserved_bytes_peak": 121993428992, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T13:14:24.784065Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115695169024, + "allocated_bytes_peak_delta": 23643729408, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 92755963904, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 92051439616, + "device": 7, + "inactive_split_bytes_current": 8182399488, + "inactive_split_bytes_peak": 11172406272, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 99870572544, + "reserved_bytes_peak": 99870572544, + "segment_count_current": 380, + "segment_count_peak": 380, + "timestamp_utc": "2026-08-01T13:14:23.541968Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9769943552, + "inactive_split_bytes_current_delta": 1587544064, + "inactive_split_bytes_peak": 15294089728, + "inactive_split_bytes_peak_delta": 4121683456, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 121993428992, + "reserved_bytes_current_delta": 22122856448, + "reserved_bytes_peak": 121993428992, + "reserved_bytes_peak_delta": 22122856448, + "segment_count_current": 440, + "segment_count_current_delta": 60, + "segment_count_peak": 440, + "segment_count_peak_delta": 60, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T131424784485Z.pickle", + "timestamp_utc": "2026-08-01T13:14:24.895408Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T131424784485Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T131424784485Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..327c92d908ce0ed902101a2a664c47e2a6c94b33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T131424784485Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dddd4bd7c7f62ca5ea381a7350b467e5bb1ae18e2e0428da1d41e5ac4f3e9337 +size 6072223 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T131426102007Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T131426102007Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ceb478ac544d82edd4922d46a13b5f98ad61361e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T131426102007Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72507620864, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116768164352, + "active_bytes_peak_delta": 368352256, + "after": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 116768164352, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 115711199744, + "device": 7, + "inactive_split_bytes_current": 10359243264, + "inactive_split_bytes_peak": 16759354880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 121993428992, + "reserved_bytes_peak": 121993428992, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T13:14:26.101585Z", + "world_size": 8 + }, + "allocated_bytes_current": 72507620864, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115711199744, + "allocated_bytes_peak_delta": 16030720, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 116399812096, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 115695169024, + "device": 7, + "inactive_split_bytes_current": 9769943552, + "inactive_split_bytes_peak": 15294089728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 121993428992, + "reserved_bytes_peak": 121993428992, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T13:14:24.911647Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10359243264, + "inactive_split_bytes_current_delta": 589299712, + "inactive_split_bytes_peak": 16759354880, + "inactive_split_bytes_peak_delta": 1465265152, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 121993428992, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121993428992, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 440, + "segment_count_current_delta": 0, + "segment_count_peak": 440, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T131426102007Z.pickle", + "timestamp_utc": "2026-08-01T13:14:26.213696Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T131426102007Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T131426102007Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c56305812aa6a4e881b9b506f4d2c10904223f13 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T131426102007Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53b20a66722c61e51871ad1765b977b7ade52a58a46bc7cf7bfa2dc6e3851ce4 +size 6592679 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T131427563882Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T131427563882Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4665f6ab3558bcfb9779583f25c296b6c88035e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T131427563882Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 162082938368, + "active_bytes_current_delta": 89575317504, + "active_bytes_peak": 162082938368, + "active_bytes_peak_delta": 45314774016, + "after": { + "active_bytes_current": 162082938368, + "active_bytes_peak": 162082938368, + "allocated_bytes_current": 106230775296, + "allocated_bytes_peak": 115711199744, + "device": 7, + "inactive_split_bytes_current": 14809735680, + "inactive_split_bytes_peak": 21646705152, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 1, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 182815031296, + "reserved_bytes_peak": 182815031296, + "segment_count_current": 665, + "segment_count_peak": 665, + "timestamp_utc": "2026-08-01T13:14:27.563287Z", + "world_size": 8 + }, + "allocated_bytes_current": 106230775296, + "allocated_bytes_current_delta": 33723154432, + "allocated_bytes_peak": 115711199744, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72507620864, + "active_bytes_peak": 116768164352, + "allocated_bytes_current": 72507620864, + "allocated_bytes_peak": 115711199744, + "device": 7, + "inactive_split_bytes_current": 10359243264, + "inactive_split_bytes_peak": 16759354880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 121993428992, + "reserved_bytes_peak": 121993428992, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T13:14:26.229572Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 14809735680, + "inactive_split_bytes_current_delta": 4450492416, + "inactive_split_bytes_peak": 21646705152, + "inactive_split_bytes_peak_delta": 4887350272, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 1, + "num_ooms_delta": 1, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 182815031296, + "reserved_bytes_current_delta": 60821602304, + "reserved_bytes_peak": 182815031296, + "reserved_bytes_peak_delta": 60821602304, + "segment_count_current": 665, + "segment_count_current_delta": 225, + "segment_count_peak": 665, + "segment_count_peak_delta": 225, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T131427563882Z.pickle", + "timestamp_utc": "2026-08-01T13:14:27.664236Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T131427563882Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T131427563882Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..590176633e70ca9871065b6002916647fefd1cae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T131427563882Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c34d937239071ed492f5242618125bb6b0e7bdd9ff82eb4e5c7268466368679e +size 5325112 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..d9f7e344b0dafef772ebf58f751621e0a29d6848 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 2048 +capacity: 1.00 +expert_dispatch_overlap: off +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6024 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 226 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T131642597239Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T131642597239Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bf34697e689f7d5020d4e5b79b16de94a567567b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T131642597239Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 31623011328, + "active_bytes_peak": 79671528448, + "active_bytes_peak_delta": 44205192192, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 79671528448, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 79671528448, + "device": 0, + "inactive_split_bytes_current": 4150905856, + "inactive_split_bytes_peak": 5205835776, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 94361354240, + "reserved_bytes_peak": 94361354240, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:16:42.596786Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 31623011328, + "allocated_bytes_peak": 79671528448, + "allocated_bytes_peak_delta": 44205192192, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:15:34.555626Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4150905856, + "inactive_split_bytes_current_delta": 4015989760, + "inactive_split_bytes_peak": 5205835776, + "inactive_split_bytes_peak_delta": 5045203968, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 94361354240, + "reserved_bytes_current_delta": 55115251712, + "reserved_bytes_peak": 94361354240, + "reserved_bytes_peak_delta": 55115251712, + "segment_count_current": 523, + "segment_count_current_delta": 451, + "segment_count_peak": 523, + "segment_count_peak_delta": 451, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T131642597239Z.pickle", + "timestamp_utc": "2026-08-01T13:16:42.709728Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T131642597239Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T131642597239Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e8e88bf4ef1998098123787765936c37b09d2dbb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T131642597239Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd8137955d4b5ccfc268583524c40f679fee866275dfbd40baeb0439baf7f14a +size 5384639 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T131643592659Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T131643592659Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d413676ea98ea8e0d6c92aebb8644d8f1ec24a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T131643592659Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103299699200, + "active_bytes_peak_delta": 23628170752, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103299699200, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103299699200, + "device": 0, + "inactive_split_bytes_current": 3515468800, + "inactive_split_bytes_peak": 5205835776, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:43.592146Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103299699200, + "allocated_bytes_peak_delta": 23628170752, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 79671528448, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 79671528448, + "device": 0, + "inactive_split_bytes_current": 4150905856, + "inactive_split_bytes_peak": 5205835776, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 94361354240, + "reserved_bytes_peak": 94361354240, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:16:42.727037Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3515468800, + "inactive_split_bytes_current_delta": -635437056, + "inactive_split_bytes_peak": 5205835776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_current_delta": 10053746688, + "reserved_bytes_peak": 104415100928, + "reserved_bytes_peak_delta": 10053746688, + "segment_count_current": 593, + "segment_count_current_delta": 70, + "segment_count_peak": 593, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T131643592659Z.pickle", + "timestamp_utc": "2026-08-01T13:16:43.706609Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T131643592659Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T131643592659Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3c0e05769d57b90e5298c019f3030e4b0837fcec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T131643592659Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a87ce9f3d4ab8496e61ab33f8875cdc1683818b0d2ef845d1d3a725f246f8745 +size 5616965 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T131644592509Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T131644592509Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4c46823dee24a8871a573457566ae4c9f3b5bcb9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T131644592509Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103300000768, + "active_bytes_peak_delta": 301568, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103300000768, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103300000768, + "device": 0, + "inactive_split_bytes_current": 3746155520, + "inactive_split_bytes_peak": 5205835776, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:44.591994Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103300000768, + "allocated_bytes_peak_delta": 301568, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103299699200, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103299699200, + "device": 0, + "inactive_split_bytes_current": 3515468800, + "inactive_split_bytes_peak": 5205835776, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:43.720126Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3746155520, + "inactive_split_bytes_current_delta": 230686720, + "inactive_split_bytes_peak": 5205835776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104415100928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 593, + "segment_count_current_delta": 0, + "segment_count_peak": 593, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T131644592509Z.pickle", + "timestamp_utc": "2026-08-01T13:16:44.696211Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T131644592509Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T131644592509Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..45d9a94dc6c643d007c4b40446cbbbd146ff3de5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T131644592509Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e2a854f022eb58a2e1904eeabc6d480f8f5ac535f53352ce082c0412c3a5306 +size 5619640 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T131645587417Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T131645587417Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b339c7aaef9d26109d524c2169de088d6cdb92d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T131645587417Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103300000768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103300000768, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103300000768, + "device": 0, + "inactive_split_bytes_current": 3987328000, + "inactive_split_bytes_peak": 5205835776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:45.586922Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103300000768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103300000768, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103300000768, + "device": 0, + "inactive_split_bytes_current": 3746155520, + "inactive_split_bytes_peak": 5205835776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:44.709060Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3987328000, + "inactive_split_bytes_current_delta": 241172480, + "inactive_split_bytes_peak": 5205835776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104415100928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 593, + "segment_count_current_delta": 0, + "segment_count_peak": 593, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T131645587417Z.pickle", + "timestamp_utc": "2026-08-01T13:16:45.686398Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T131645587417Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T131645587417Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..50092c997f81875856efbe00b0b786762e7cb851 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T131645587417Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d577e58e0fbc508220b21c46e7c0197a1e80720609c3f15750105a2af8141088 +size 5618931 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T131647146333Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T131647146333Z.json new file mode 100644 index 0000000000000000000000000000000000000000..05e5835cede7fe4adcdb61b0a90e5111f23b4e9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T131647146333Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103301805056, + "active_bytes_peak_delta": 1804288, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103301805056, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103301805056, + "device": 0, + "inactive_split_bytes_current": 3985230848, + "inactive_split_bytes_peak": 5205835776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:47.145840Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103301805056, + "allocated_bytes_peak_delta": 1804288, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103300000768, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103300000768, + "device": 0, + "inactive_split_bytes_current": 3987328000, + "inactive_split_bytes_peak": 5205835776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:45.698626Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3985230848, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 5205835776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104415100928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 593, + "segment_count_current_delta": 0, + "segment_count_peak": 593, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T131647146333Z.pickle", + "timestamp_utc": "2026-08-01T13:16:47.251441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T131647146333Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T131647146333Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..96da77216a2a26d656665380721946132c8938e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T131647146333Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:107820dfae6a287879e181eaa6097742b96a2f26dc3767f3aa028f07ca40a747 +size 5618990 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T131648138145Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T131648138145Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5dafef1686e7e9031ca32e40ffa53a2e3d837736 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T131648138145Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103302464512, + "active_bytes_peak_delta": 659456, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103302464512, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103302464512, + "device": 0, + "inactive_split_bytes_current": 3960065024, + "inactive_split_bytes_peak": 5205835776, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:48.137685Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103302464512, + "allocated_bytes_peak_delta": 659456, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103301805056, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103301805056, + "device": 0, + "inactive_split_bytes_current": 3985230848, + "inactive_split_bytes_peak": 5205835776, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:47.265890Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3960065024, + "inactive_split_bytes_current_delta": -25165824, + "inactive_split_bytes_peak": 5205835776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104415100928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 593, + "segment_count_current_delta": 0, + "segment_count_peak": 593, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T131648138145Z.pickle", + "timestamp_utc": "2026-08-01T13:16:48.237865Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T131648138145Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T131648138145Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..66bbdc6b236940e0a9afc0121ef83045fe9c1bfe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T131648138145Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0d4bbcc6fba0108cfd3f0a5f0d2c54902c8069ba4de1b5452dcbc4dfd21c825 +size 5618787 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T131649115344Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T131649115344Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c68ac8539b32cbb5de1b5d286f420b87378a71ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T131649115344Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103302464512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103302464512, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103302464512, + "device": 0, + "inactive_split_bytes_current": 3955870720, + "inactive_split_bytes_peak": 5205835776, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:49.114851Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103302464512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103302464512, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103302464512, + "device": 0, + "inactive_split_bytes_current": 3960065024, + "inactive_split_bytes_peak": 5205835776, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:48.249297Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3955870720, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 5205835776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104415100928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 593, + "segment_count_current_delta": 0, + "segment_count_peak": 593, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T131649115344Z.pickle", + "timestamp_utc": "2026-08-01T13:16:49.210976Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T131649115344Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T131649115344Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da5f4c34b0a7de6a5590994c832015a4f11230c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T131649115344Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3867c58ce0b6d7f2ee618b1f76f6f05d5b05ab0bee6b33b06df927af35ff7cb +size 5618474 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T131650099901Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T131650099901Z.json new file mode 100644 index 0000000000000000000000000000000000000000..937bb38f2311206854519bcb03c4baa6c73d2933 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T131650099901Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103302464512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103302464512, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103302464512, + "device": 0, + "inactive_split_bytes_current": 3955870720, + "inactive_split_bytes_peak": 5205835776, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:50.099385Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103302464512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103302464512, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103302464512, + "device": 0, + "inactive_split_bytes_current": 3955870720, + "inactive_split_bytes_peak": 5205835776, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:49.222558Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3955870720, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5205835776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104415100928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 593, + "segment_count_current_delta": 0, + "segment_count_peak": 593, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T131650099901Z.pickle", + "timestamp_utc": "2026-08-01T13:16:50.199876Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T131650099901Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T131650099901Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..64ec174ca5e370d81a9ce262bbfddaa3c8668e4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T131650099901Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b89f7362c65c5ca7bb13b12d6ad53f547a4f9756985ab47a831329dadc9a5efe +size 5618425 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T131651082252Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T131651082252Z.json new file mode 100644 index 0000000000000000000000000000000000000000..240c01767bd09a41102ad0878785c53c879f5d3e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T131651082252Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103302464512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103302464512, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103302464512, + "device": 0, + "inactive_split_bytes_current": 3957967872, + "inactive_split_bytes_peak": 5205835776, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:51.081763Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103302464512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103302464512, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103302464512, + "device": 0, + "inactive_split_bytes_current": 3955870720, + "inactive_split_bytes_peak": 5205835776, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:50.212068Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3957967872, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 5205835776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104415100928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 593, + "segment_count_current_delta": 0, + "segment_count_peak": 593, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T131651082252Z.pickle", + "timestamp_utc": "2026-08-01T13:16:51.176128Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T131651082252Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T131651082252Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93867cfa90aa1a6670e93729e88c06b906176193 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T131651082252Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3eed3c69517b8785bbcc5bf489b0f0e0e8ae9778ac71878536316e450cc2fc8e +size 5618363 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T131749814243Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T131749814243Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a996322ab7f58ebc3d66f968058a54587f8368e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T131749814243Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803823616, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103302464512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803823616, + "active_bytes_peak": 103302464512, + "allocated_bytes_current": 66803823616, + "allocated_bytes_peak": 103302464512, + "device": 0, + "inactive_split_bytes_current": 3031337984, + "inactive_split_bytes_peak": 5205835776, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:17:49.813772Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803823616, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103302464512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103302464512, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103302464512, + "device": 0, + "inactive_split_bytes_current": 3957967872, + "inactive_split_bytes_peak": 5205835776, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_peak": 104415100928, + "segment_count_current": 593, + "segment_count_peak": 593, + "timestamp_utc": "2026-08-01T13:16:51.187493Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3031337984, + "inactive_split_bytes_current_delta": -926629888, + "inactive_split_bytes_peak": 5205835776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104415100928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104415100928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 593, + "segment_count_current_delta": 0, + "segment_count_peak": 593, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T131749814243Z.pickle", + "timestamp_utc": "2026-08-01T13:17:50.009147Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T131749814243Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T131749814243Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7c52135f0ebffb4825358d99ba30e1a1d7b44bcd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T131749814243Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6115dc9d39f175dc27a6027a8fcfea73ed8ee90d00d6d47feaa58c76aca0bdea +size 10196670 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T131642597041Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T131642597041Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2b70d40734d0d70dcdebda1d05d75dd1c5843b61 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T131642597041Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 79676544000, + "active_bytes_peak_delta": 44210207744, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 79676544000, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 79676544000, + "device": 1, + "inactive_split_bytes_current": 4265069568, + "inactive_split_bytes_peak": 5403768832, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 93889495040, + "reserved_bytes_peak": 93889495040, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:16:42.596455Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 79676544000, + "allocated_bytes_peak_delta": 44210207744, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:15:34.555621Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4265069568, + "inactive_split_bytes_current_delta": 4132250624, + "inactive_split_bytes_peak": 5403768832, + "inactive_split_bytes_peak_delta": 5243382784, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 93889495040, + "reserved_bytes_current_delta": 54645489664, + "reserved_bytes_peak": 93889495040, + "reserved_bytes_peak_delta": 54645489664, + "segment_count_current": 519, + "segment_count_current_delta": 448, + "segment_count_peak": 519, + "segment_count_peak_delta": 448, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T131642597041Z.pickle", + "timestamp_utc": "2026-08-01T13:16:42.712066Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T131642597041Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T131642597041Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cfd2fa6dbc84ffb47cda72820db075ebbde7872a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T131642597041Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7e806394f562feaf59e4c8c9ae885446d0ac89ad4fbe7ab67a7cbc19bfc8c2f +size 5379867 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T131643592484Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T131643592484Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3b005a8dd5cc6183903fa0698519c277350630d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T131643592484Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 23645672448, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3617049600, + "inactive_split_bytes_peak": 5403768832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:43.592078Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 23645672448, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 79676544000, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 79676544000, + "device": 1, + "inactive_split_bytes_current": 4265069568, + "inactive_split_bytes_peak": 5403768832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 93889495040, + "reserved_bytes_peak": 93889495040, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:16:42.731225Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3617049600, + "inactive_split_bytes_current_delta": -648019968, + "inactive_split_bytes_peak": 5403768832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 10519314432, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 10519314432, + "segment_count_current": 591, + "segment_count_current_delta": 72, + "segment_count_peak": 591, + "segment_count_peak_delta": 72, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T131643592484Z.pickle", + "timestamp_utc": "2026-08-01T13:16:43.704627Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T131643592484Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T131643592484Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c89d3daa57d481488f562ac167fa220c1ec5e644 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T131643592484Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01901a2e285426a0b2dbc3f48785c33844a88aa6408a0c0d1d442fe3b40e269f +size 5613023 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T131644592286Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T131644592286Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1550e0898f643ae36d52d0575e4babf14352e7f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T131644592286Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3677867008, + "inactive_split_bytes_peak": 5403768832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:44.591889Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3617049600, + "inactive_split_bytes_peak": 5403768832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:43.719935Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3677867008, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 5403768832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 591, + "segment_count_current_delta": 0, + "segment_count_peak": 591, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T131644592286Z.pickle", + "timestamp_utc": "2026-08-01T13:16:44.696777Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T131644592286Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T131644592286Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..371aa6ed96ecb976a0644fab30187d85c09d9004 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T131644592286Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7503409d66569daabf4506fd18f65229173c3e72857dd6286c2de13307db65a7 +size 5614124 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T131645587267Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T131645587267Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5c5bfe1a6abac6920f3a7c3fd2f5011ec863b5a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T131645587267Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3912748032, + "inactive_split_bytes_peak": 5403768832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:45.586830Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3677867008, + "inactive_split_bytes_peak": 5403768832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:44.711609Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3912748032, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 5403768832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 591, + "segment_count_current_delta": 0, + "segment_count_peak": 591, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T131645587267Z.pickle", + "timestamp_utc": "2026-08-01T13:16:45.691954Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T131645587267Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T131645587267Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7a6c498f0a158c6ec01a60eb57efa48226d06d32 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T131645587267Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:287d1a025fb56c7e5163103e51b03206a76209a0a645404b9e4db5e3e57a2257 +size 5613891 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T131647146118Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T131647146118Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d3e8697ad103d6e7986d6ad45b859a3fe5815e4b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T131647146118Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3797404672, + "inactive_split_bytes_peak": 5403768832, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:47.145707Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3912748032, + "inactive_split_bytes_peak": 5403768832, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:45.706823Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3797404672, + "inactive_split_bytes_current_delta": -115343360, + "inactive_split_bytes_peak": 5403768832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 591, + "segment_count_current_delta": 0, + "segment_count_peak": 591, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T131647146118Z.pickle", + "timestamp_utc": "2026-08-01T13:16:47.255367Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T131647146118Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T131647146118Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0cca9fb3b9a084e65f5df029a7723f3f8e66eb27 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T131647146118Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff9b0a13787d7267b4282d5db702162df15f2c3519ce109aec698e98e761c1c3 +size 5614042 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T131648137981Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T131648137981Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9df6bcaefe6e2222d36398502fa8141a12741f54 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T131648137981Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3908553728, + "inactive_split_bytes_peak": 5403768832, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:48.137586Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3797404672, + "inactive_split_bytes_peak": 5403768832, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:47.272330Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3908553728, + "inactive_split_bytes_current_delta": 111149056, + "inactive_split_bytes_peak": 5403768832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 591, + "segment_count_current_delta": 0, + "segment_count_peak": 591, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T131648137981Z.pickle", + "timestamp_utc": "2026-08-01T13:16:48.238922Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T131648137981Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T131648137981Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..807625daf5a5f65ff09175c3af3ae7d280b32b46 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T131648137981Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4855bdd84b2511342fd6edd06ba503c769232ab8f4b6679d8b6916645c145aa9 +size 5613048 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T131649115144Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T131649115144Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7091faf27b5d361b80b6c1b4b30b3460be0fd8b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T131649115144Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3795307520, + "inactive_split_bytes_peak": 5403768832, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:49.114742Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3908553728, + "inactive_split_bytes_peak": 5403768832, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:48.253211Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3795307520, + "inactive_split_bytes_current_delta": -113246208, + "inactive_split_bytes_peak": 5403768832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 591, + "segment_count_current_delta": 0, + "segment_count_peak": 591, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T131649115144Z.pickle", + "timestamp_utc": "2026-08-01T13:16:49.217489Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T131649115144Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T131649115144Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2eac5c8d52604ba858e1c16df44fc8d5446891e1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T131649115144Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:571ec30155852044de1a2583dfefc55d1c6460aec1750bcfdf6f2eae84c5e707 +size 5613071 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T131650099668Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T131650099668Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b83d5b9972e6813b75cda044795ebec97e7ce423 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T131650099668Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3912748032, + "inactive_split_bytes_peak": 5403768832, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:50.099267Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3795307520, + "inactive_split_bytes_peak": 5403768832, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:49.231776Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3912748032, + "inactive_split_bytes_current_delta": 117440512, + "inactive_split_bytes_peak": 5403768832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 591, + "segment_count_current_delta": 0, + "segment_count_peak": 591, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T131650099668Z.pickle", + "timestamp_utc": "2026-08-01T13:16:50.198954Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T131650099668Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T131650099668Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..af79dde729bee79d7c14a0d431a74baa10fcc82a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T131650099668Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:676f338c32d61c52c54bb86059e4f3431d2f2a282b6b1afecf7a285a7127196d +size 5612985 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T131651082095Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T131651082095Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ba9f20ba6a6dc9e87a0807aa4dcfa4eedf91266c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T131651082095Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3789016064, + "inactive_split_bytes_peak": 5403768832, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:51.081690Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3912748032, + "inactive_split_bytes_peak": 5403768832, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:50.212980Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3789016064, + "inactive_split_bytes_current_delta": -123731968, + "inactive_split_bytes_peak": 5403768832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 591, + "segment_count_current_delta": 0, + "segment_count_peak": 591, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T131651082095Z.pickle", + "timestamp_utc": "2026-08-01T13:16:51.180305Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T131651082095Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T131651082095Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f129e2e57d1535857d7a5dcb34c0894a3a3bc54b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T131651082095Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29b4f13a345a0d53a53470cc89f68b8cda75ba837cfff208ad5157e278babac2 +size 5612943 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T131749814265Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T131749814265Z.json new file mode 100644 index 0000000000000000000000000000000000000000..99d8aca993f16134075d04481b65b7e0fbe0022b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T131749814265Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3105655808, + "inactive_split_bytes_peak": 5403768832, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:17:49.813769Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 1, + "inactive_split_bytes_current": 3789016064, + "inactive_split_bytes_peak": 5403768832, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 591, + "segment_count_peak": 591, + "timestamp_utc": "2026-08-01T13:16:51.194489Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3105655808, + "inactive_split_bytes_current_delta": -683360256, + "inactive_split_bytes_peak": 5403768832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 591, + "segment_count_current_delta": 0, + "segment_count_peak": 591, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T131749814265Z.pickle", + "timestamp_utc": "2026-08-01T13:17:50.033733Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T131749814265Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T131749814265Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2599a60b589fee7f2e70eeae239f7427f0492136 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T131749814265Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a86640e28356bd054f24a35b8e96e9a54e96610a4e016dd6f52ea4629737c02 +size 9811356 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T131642596942Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T131642596942Z.json new file mode 100644 index 0000000000000000000000000000000000000000..275c6462452a5ed20a48c4ca31728b99626d20f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T131642596942Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 79676544000, + "active_bytes_peak_delta": 44210207744, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 79676544000, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 79676544000, + "device": 2, + "inactive_split_bytes_current": 4028091392, + "inactive_split_bytes_peak": 5674427392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93419732992, + "reserved_bytes_peak": 93419732992, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:16:42.596343Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 79676544000, + "allocated_bytes_peak_delta": 44210207744, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:15:34.555548Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4028091392, + "inactive_split_bytes_current_delta": 3895272448, + "inactive_split_bytes_peak": 5674427392, + "inactive_split_bytes_peak_delta": 5514041344, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93419732992, + "reserved_bytes_current_delta": 54175727616, + "reserved_bytes_peak": 93419732992, + "reserved_bytes_peak_delta": 54175727616, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T131642596942Z.pickle", + "timestamp_utc": "2026-08-01T13:16:42.705436Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T131642596942Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T131642596942Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..353d26e9c5357cab904f38007f4aeb027789d69f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T131642596942Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a498eba1465bcf78e3351c58277d445a2754b920498ccd6faad2526086eee2a5 +size 5376052 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T131643592511Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T131643592511Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c1de36a7c67a7b4ac287965c966f24ededf92d84 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T131643592511Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 23643837440, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3556232192, + "inactive_split_bytes_peak": 5674427392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:43.592077Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 23643837440, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 79676544000, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 79676544000, + "device": 2, + "inactive_split_bytes_current": 4028091392, + "inactive_split_bytes_peak": 5674427392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 93419732992, + "reserved_bytes_peak": 93419732992, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:16:42.722879Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3556232192, + "inactive_split_bytes_current_delta": -471859200, + "inactive_split_bytes_peak": 5674427392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 10989076480, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 10989076480, + "segment_count_current": 590, + "segment_count_current_delta": 78, + "segment_count_peak": 590, + "segment_count_peak_delta": 78, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T131643592511Z.pickle", + "timestamp_utc": "2026-08-01T13:16:43.703031Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T131643592511Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T131643592511Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0852bff1bf0030862b168ac0de3e10ba9948dec8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T131643592511Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec7b3f3298ded869f9b447ce2f676cb861b250c9d953c92376b5603d2ccb447d +size 5611462 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T131644592227Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T131644592227Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c0a15c1fe23fb75385436e743961f9a7238f531e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T131644592227Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3677867008, + "inactive_split_bytes_peak": 5674427392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:44.591824Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3556232192, + "inactive_split_bytes_peak": 5674427392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:43.716527Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3677867008, + "inactive_split_bytes_current_delta": 121634816, + "inactive_split_bytes_peak": 5674427392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T131644592227Z.pickle", + "timestamp_utc": "2026-08-01T13:16:44.694713Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T131644592227Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T131644592227Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3e2214c935f1ef7c3e4397f676ca877d2515734 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T131644592227Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fadf7122ca137f9c2d7c33343d5236ca7d3a0dcc2b2be5da7ddc85a7d941479e +size 5615421 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T131645587171Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T131645587171Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80cbcfeb67796d60eec82641d5291c0fa70e86f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T131645587171Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_peak": 5674427392, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:45.586775Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3677867008, + "inactive_split_bytes_peak": 5674427392, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:44.707796Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 5674427392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T131645587171Z.pickle", + "timestamp_utc": "2026-08-01T13:16:45.688608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T131645587171Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T131645587171Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0de97de1be84ef4352bf465acfd049f0024287af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T131645587171Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42e8550c847ac77db17a4873398eb739e840025a4eebd20b7e58f69ee56de973 +size 5613674 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T131647146107Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T131647146107Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c13cef896d3cf9dd3a227b29a021cc471480db6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T131647146107Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3562523648, + "inactive_split_bytes_peak": 5674427392, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:47.145690Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_peak": 5674427392, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:45.701624Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3562523648, + "inactive_split_bytes_current_delta": -113246208, + "inactive_split_bytes_peak": 5674427392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T131647146107Z.pickle", + "timestamp_utc": "2026-08-01T13:16:47.249081Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T131647146107Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T131647146107Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..56ff16fb3ba2775053961a7ea76eaff10cefc0ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T131647146107Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b13d5b8ad7717b23e21a128ab26a6279a7c5a8afc8d9938aa3c30b4fa30485d +size 5613868 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T131648137982Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T131648137982Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc355387a464b2e59ef1b9743636a6f92f514e9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T131648137982Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3673672704, + "inactive_split_bytes_peak": 5674427392, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:48.137587Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3562523648, + "inactive_split_bytes_peak": 5674427392, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:47.262135Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3673672704, + "inactive_split_bytes_current_delta": 111149056, + "inactive_split_bytes_peak": 5674427392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T131648137982Z.pickle", + "timestamp_utc": "2026-08-01T13:16:48.241857Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T131648137982Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T131648137982Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dffc4b58cc81298ea91aa1bf47167c08ed7efc65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T131648137982Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e37b54697f10a37377b93df8147bffb3c16e013a0be017d57c721cd335fc563 +size 5613373 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T131649115118Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T131649115118Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18163cea346b4d565a3146649812d136ecb62081 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T131649115118Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3560426496, + "inactive_split_bytes_peak": 5674427392, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:49.114715Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3673672704, + "inactive_split_bytes_peak": 5674427392, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:48.256791Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3560426496, + "inactive_split_bytes_current_delta": -113246208, + "inactive_split_bytes_peak": 5674427392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T131649115118Z.pickle", + "timestamp_utc": "2026-08-01T13:16:49.213103Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T131649115118Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T131649115118Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5681cb63fef15ef684b836b309bebda70382c87e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T131649115118Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:592f15fb134467c23234044bc5cc54402ee4663ce3c2e92c69be7418fa3685b8 +size 5613439 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T131650099672Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T131650099672Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3f9e29be5ba4b4875998c8fb81b932297453ae6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T131650099672Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3669478400, + "inactive_split_bytes_peak": 5674427392, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:50.099261Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3560426496, + "inactive_split_bytes_peak": 5674427392, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:49.225633Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3669478400, + "inactive_split_bytes_current_delta": 109051904, + "inactive_split_bytes_peak": 5674427392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T131650099672Z.pickle", + "timestamp_utc": "2026-08-01T13:16:50.199516Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T131650099672Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T131650099672Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2b22c858480fcddf3418f70a578ca3111cba9c0a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T131650099672Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8fd11da9c141f92e0edeff8deb72df1026c373dbb52646fad58bbcf34ae0bec +size 5613105 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T131651082096Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T131651082096Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6152739e2655c255cf6dbf3cbe9f9d22de9e292 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T131651082096Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3562523648, + "inactive_split_bytes_peak": 5674427392, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:51.081672Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3669478400, + "inactive_split_bytes_peak": 5674427392, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:50.212259Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3562523648, + "inactive_split_bytes_current_delta": -106954752, + "inactive_split_bytes_peak": 5674427392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T131651082096Z.pickle", + "timestamp_utc": "2026-08-01T13:16:51.179014Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T131651082096Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T131651082096Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..739063e2d6fad6c65f472d85cff5cb2640d8502a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T131651082096Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a3c49362407f5b81158cc0bf904112b978c6023345ef1b70f70ebafae69a287 +size 5613207 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T131749814343Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T131749814343Z.json new file mode 100644 index 0000000000000000000000000000000000000000..753d05e292c6a64a464b24a48382cff594cf25b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T131749814343Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3103558656, + "inactive_split_bytes_peak": 5674427392, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:17:49.813778Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 2, + "inactive_split_bytes_current": 3562523648, + "inactive_split_bytes_peak": 5674427392, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:51.191399Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3103558656, + "inactive_split_bytes_current_delta": -458964992, + "inactive_split_bytes_peak": 5674427392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T131749814343Z.pickle", + "timestamp_utc": "2026-08-01T13:17:50.009146Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T131749814343Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T131749814343Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c157b30fc6c9dd6e6c2acd86a8c206ee5620ede0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T131749814343Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:171385429a5b9bb7d0d563c486be234402637ccdd797bbaa3d6692b4d10d815d +size 9754793 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T131642596906Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T131642596906Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03baf0bedc2a1a3e6a6cdd247832348023558ab3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T131642596906Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 79676544000, + "active_bytes_peak_delta": 44210207744, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 79676544000, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 79676544000, + "device": 3, + "inactive_split_bytes_current": 4032285696, + "inactive_split_bytes_peak": 5766865920, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93419732992, + "reserved_bytes_peak": 93419732992, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:16:42.596341Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 79676544000, + "allocated_bytes_peak_delta": 44210207744, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:15:34.555564Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4032285696, + "inactive_split_bytes_current_delta": 3899466752, + "inactive_split_bytes_peak": 5766865920, + "inactive_split_bytes_peak_delta": 5606479872, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93419732992, + "reserved_bytes_current_delta": 54175727616, + "reserved_bytes_peak": 93419732992, + "reserved_bytes_peak_delta": 54175727616, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T131642596906Z.pickle", + "timestamp_utc": "2026-08-01T13:16:42.706007Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T131642596906Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T131642596906Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f888f6109759955224b1c44e1005fa4982d24eb2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T131642596906Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab43b9de85b7b14a8369625f993066e07274a538b8a45f44ce0258f4affc343b +size 5378660 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T131643592476Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T131643592476Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c1e4ac4ae1f7cfcf16877feb227a5d92be5075a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T131643592476Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 23643837440, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3552037888, + "inactive_split_bytes_peak": 5766865920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:43.592068Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 23643837440, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 79676544000, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 79676544000, + "device": 3, + "inactive_split_bytes_current": 4032285696, + "inactive_split_bytes_peak": 5766865920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93419732992, + "reserved_bytes_peak": 93419732992, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:16:42.722906Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3552037888, + "inactive_split_bytes_current_delta": -480247808, + "inactive_split_bytes_peak": 5766865920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 10989076480, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 10989076480, + "segment_count_current": 590, + "segment_count_current_delta": 78, + "segment_count_peak": 590, + "segment_count_peak_delta": 78, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T131643592476Z.pickle", + "timestamp_utc": "2026-08-01T13:16:43.700883Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T131643592476Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T131643592476Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a232187a18cdadba6d6583b07a378f7f4815afe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T131643592476Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1322ba5b493cdd8b2100fddb73da8ccb3c7905cad2c65df666f74a955e952f9d +size 5611677 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T131644592276Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T131644592276Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6b551dfd8247ebfb21cf0907c056789a0092eadc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T131644592276Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3677867008, + "inactive_split_bytes_peak": 5766865920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:44.591857Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3552037888, + "inactive_split_bytes_peak": 5766865920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:43.713949Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3677867008, + "inactive_split_bytes_current_delta": 125829120, + "inactive_split_bytes_peak": 5766865920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T131644592276Z.pickle", + "timestamp_utc": "2026-08-01T13:16:44.693563Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T131644592276Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T131644592276Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf8c80efe2888181d11732e122a399ef28fafcc4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T131644592276Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f8162dc322e3a085bf096ad58535d7944bb1ff98b7256c8a5cd073e5b8db101 +size 5613918 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T131645587244Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T131645587244Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70e2d6ad3d9e731efd90667671fbeb0d3acdc227 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T131645587244Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_peak": 5766865920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:45.586816Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3677867008, + "inactive_split_bytes_peak": 5766865920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:44.706296Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 5766865920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T131645587244Z.pickle", + "timestamp_utc": "2026-08-01T13:16:45.689737Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T131645587244Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T131645587244Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f336a55818f63672f3d71595903c6c7f470de97d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T131645587244Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2c20dd44a91e1afe001fa8ca4b6b32aee1ed4fc42291aba71a68556e7f2ccf4 +size 5614156 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T131647146105Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T131647146105Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac87d17f593daf76a7730fa4b065d15ee121bb51 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T131647146105Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3560426496, + "inactive_split_bytes_peak": 5766865920, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:47.145708Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_peak": 5766865920, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:45.702492Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3560426496, + "inactive_split_bytes_current_delta": -115343360, + "inactive_split_bytes_peak": 5766865920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T131647146105Z.pickle", + "timestamp_utc": "2026-08-01T13:16:47.255140Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T131647146105Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T131647146105Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..74c75b1b9cecbd0b2a6697746c73e584c6f3f985 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T131647146105Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b9f8802785c265e52333cffcb38b11971fc9ad24a7450018e1057dc3b95775e +size 5613997 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T131648137992Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T131648137992Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cb1818d053386649b303e09411d38b583e66b738 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T131648137992Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3677867008, + "inactive_split_bytes_peak": 5766865920, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:48.137591Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3560426496, + "inactive_split_bytes_peak": 5766865920, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:47.270650Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3677867008, + "inactive_split_bytes_current_delta": 117440512, + "inactive_split_bytes_peak": 5766865920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T131648137992Z.pickle", + "timestamp_utc": "2026-08-01T13:16:48.236060Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T131648137992Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T131648137992Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..09fc2410e6c716b2b3a3ad2fdb5b036e52b31a45 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T131648137992Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac01a094e80789a350ecb8a19b60f00383036389d7982551d2cf963070436bba +size 5614297 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T131649115157Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T131649115157Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f7612dda38b9bea19c0ed41403a4624a66bbf3b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T131649115157Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3562523648, + "inactive_split_bytes_peak": 5766865920, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:49.114756Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3677867008, + "inactive_split_bytes_peak": 5766865920, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:48.248642Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3562523648, + "inactive_split_bytes_current_delta": -115343360, + "inactive_split_bytes_peak": 5766865920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T131649115157Z.pickle", + "timestamp_utc": "2026-08-01T13:16:49.212715Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T131649115157Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T131649115157Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..281cf947a7e24a5d12102919c88595f51423d996 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T131649115157Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be9e55f600cac255c95adae61f474aca2aa437b23040f7a0fc1e26ac1db52696 +size 5614444 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T131650099659Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T131650099659Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aca6ddc804756017896ae1d3f3a1d61ecd9d0e3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T131650099659Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3665284096, + "inactive_split_bytes_peak": 5766865920, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:50.099260Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3562523648, + "inactive_split_bytes_peak": 5766865920, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:49.225055Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3665284096, + "inactive_split_bytes_current_delta": 102760448, + "inactive_split_bytes_peak": 5766865920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T131650099659Z.pickle", + "timestamp_utc": "2026-08-01T13:16:50.198774Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T131650099659Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T131650099659Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6f0c8cdcb0bd359e9ea89856a88d5bc66bd0a5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T131650099659Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44b07990960b1298d5e1a5c164b7dd2bc8efacb0fbe7a6667429403926a5554d +size 5612978 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T131651082086Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T131651082086Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad354020a40f911ec041570a688f8bd4e3f12108 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T131651082086Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3558329344, + "inactive_split_bytes_peak": 5766865920, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:51.081681Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3665284096, + "inactive_split_bytes_peak": 5766865920, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:50.211186Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3558329344, + "inactive_split_bytes_current_delta": -106954752, + "inactive_split_bytes_peak": 5766865920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T131651082086Z.pickle", + "timestamp_utc": "2026-08-01T13:16:51.179524Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T131651082086Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T131651082086Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..49e8290d85a7906f22f267b15cb640b86d48f9d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T131651082086Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0627b8c02fe3e02ca8ace6b04d5c35f22d7cab93759cb250f6ed5dcf293af539 +size 5614701 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T131749814334Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T131749814334Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5b573d0c2bca5769cf37a486001df0bf7e3d645 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T131749814334Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3105655808, + "inactive_split_bytes_peak": 5766865920, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:17:49.813776Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 3, + "inactive_split_bytes_current": 3558329344, + "inactive_split_bytes_peak": 5766865920, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:51.191877Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3105655808, + "inactive_split_bytes_current_delta": -452673536, + "inactive_split_bytes_peak": 5766865920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T131749814334Z.pickle", + "timestamp_utc": "2026-08-01T13:17:50.010541Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T131749814334Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T131749814334Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f560fb21cd72b4384ed9eaea1a86d11058b92346 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T131749814334Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3985f8f8a7ba89c6b919037b9e7919a44ddf13f24d6b9bb422d1aa1d81bbc82 +size 9754755 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T131642596858Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T131642596858Z.json new file mode 100644 index 0000000000000000000000000000000000000000..32a2511a715da81b176eee10d5576dc174b7c6a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T131642596858Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854789120, + "active_bytes_current_delta": 31622891520, + "active_bytes_peak": 79676220416, + "active_bytes_peak_delta": 44209441792, + "after": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 79676220416, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 79676220416, + "device": 4, + "inactive_split_bytes_current": 4498710528, + "inactive_split_bytes_peak": 5872025600, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 93889495040, + "reserved_bytes_peak": 93889495040, + "segment_count_current": 513, + "segment_count_peak": 513, + "timestamp_utc": "2026-08-01T13:16:42.596444Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854789120, + "allocated_bytes_current_delta": 31622891520, + "allocated_bytes_peak": 79676220416, + "allocated_bytes_peak_delta": 44209441792, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:15:34.555097Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4498710528, + "inactive_split_bytes_current_delta": 4366333952, + "inactive_split_bytes_peak": 5872025600, + "inactive_split_bytes_peak_delta": 5718616064, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 93889495040, + "reserved_bytes_current_delta": 54645489664, + "reserved_bytes_peak": 93889495040, + "reserved_bytes_peak_delta": 54645489664, + "segment_count_current": 513, + "segment_count_current_delta": 442, + "segment_count_peak": 513, + "segment_count_peak_delta": 442, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T131642596858Z.pickle", + "timestamp_utc": "2026-08-01T13:16:42.707756Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T131642596858Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T131642596858Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dbe23f9b5803ea927cbb9f8120ad506de43e041b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T131642596858Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d30c5ed67e937d5655f644ad87003475540a2bae8e462d82480b8ccd7ab1006 +size 5383926 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T131643592435Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T131643592435Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4799d203c9b6290bff1a0a267c34ce2ae1bc5c62 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T131643592435Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854789120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103316367360, + "active_bytes_peak_delta": 23640146944, + "after": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3680821248, + "inactive_split_bytes_peak": 5872025600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:43.592025Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854789120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103316367360, + "allocated_bytes_peak_delta": 23640146944, + "before": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 79676220416, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 79676220416, + "device": 4, + "inactive_split_bytes_current": 4498710528, + "inactive_split_bytes_peak": 5872025600, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 93889495040, + "reserved_bytes_peak": 93889495040, + "segment_count_current": 513, + "segment_count_peak": 513, + "timestamp_utc": "2026-08-01T13:16:42.727208Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3680821248, + "inactive_split_bytes_current_delta": -817889280, + "inactive_split_bytes_peak": 5872025600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104383643648, + "reserved_bytes_current_delta": 10494148608, + "reserved_bytes_peak": 104383643648, + "reserved_bytes_peak_delta": 10494148608, + "segment_count_current": 587, + "segment_count_current_delta": 74, + "segment_count_peak": 587, + "segment_count_peak_delta": 74, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T131643592435Z.pickle", + "timestamp_utc": "2026-08-01T13:16:43.706188Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T131643592435Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T131643592435Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..005578d0c5b2259cae5d876b3f5e69704629b675 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T131643592435Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d67fbc775976f6efb02b719149f5fa5cdc6a8a9640b7ca1efc4482fe636dc9bb +size 5620766 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T131644592232Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T131644592232Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a16192ec85ae86748a10546496d1169e2f81107d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T131644592232Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854789120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103316367360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 4037337088, + "inactive_split_bytes_peak": 5872025600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:44.591830Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854789120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103316367360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3680821248, + "inactive_split_bytes_peak": 5872025600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:43.721453Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4037337088, + "inactive_split_bytes_current_delta": 356515840, + "inactive_split_bytes_peak": 5872025600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104383643648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104383643648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 587, + "segment_count_current_delta": 0, + "segment_count_peak": 587, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T131644592232Z.pickle", + "timestamp_utc": "2026-08-01T13:16:44.695828Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T131644592232Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T131644592232Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9015d64c5ed2337f9142186e9dd31b8cfbcfc1f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T131644592232Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70e4dbec6cadf19ba29a993a2c72382b41e6692035b932e68ee7bafa97c31b71 +size 5616134 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T131645587188Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T131645587188Z.json new file mode 100644 index 0000000000000000000000000000000000000000..331bca18b65a932de521c44a7aff09830c6a933c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T131645587188Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854789120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103316367360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3976519680, + "inactive_split_bytes_peak": 5872025600, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:45.586772Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854789120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103316367360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 4037337088, + "inactive_split_bytes_peak": 5872025600, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:44.710719Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3976519680, + "inactive_split_bytes_current_delta": -60817408, + "inactive_split_bytes_peak": 5872025600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104383643648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104383643648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 587, + "segment_count_current_delta": 0, + "segment_count_peak": 587, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T131645587188Z.pickle", + "timestamp_utc": "2026-08-01T13:16:45.691270Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T131645587188Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T131645587188Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..968de2bb10331ec0b8c0caa3abe8588152ac3593 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T131645587188Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35d348f40bdd8baf726de7b39a5b461621839d6a2896dfa6732101d9eba9878f +size 5615559 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T131647146130Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T131647146130Z.json new file mode 100644 index 0000000000000000000000000000000000000000..10254b3e537e3df4f82d464e661231d82dd16580 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T131647146130Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854789120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103316367360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3978616832, + "inactive_split_bytes_peak": 5872025600, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:47.145700Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854789120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103316367360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3976519680, + "inactive_split_bytes_peak": 5872025600, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:45.705993Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3978616832, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 5872025600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104383643648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104383643648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 587, + "segment_count_current_delta": 0, + "segment_count_peak": 587, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T131647146130Z.pickle", + "timestamp_utc": "2026-08-01T13:16:47.256696Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T131647146130Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T131647146130Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..933886a7b901d158d30bd7fcbd2d527d7c976fc4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T131647146130Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d872fe241188d336b0fbcc9724605e27017e5d6f6fecdf1c4104f537d46eacb7 +size 5621292 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T131648137942Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T131648137942Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4300f9cd8f70bf4815c6d6e89b85b72ff257e9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T131648137942Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854789120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103316367360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3976519680, + "inactive_split_bytes_peak": 5872025600, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:48.137544Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854789120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103316367360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3978616832, + "inactive_split_bytes_peak": 5872025600, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:47.273896Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3976519680, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 5872025600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104383643648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104383643648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 587, + "segment_count_current_delta": 0, + "segment_count_peak": 587, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T131648137942Z.pickle", + "timestamp_utc": "2026-08-01T13:16:48.239028Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T131648137942Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T131648137942Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..000ffce3fd2e205dd7f9353485b8c37600efda7b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T131648137942Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b157bb6e82507971dda59d5afcea5c5a3bade7b2929ba4a25ac9532a9b2c7263 +size 5615716 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T131649115111Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T131649115111Z.json new file mode 100644 index 0000000000000000000000000000000000000000..efc3a3ca4290b7504d45fc92b7665302727a5d59 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T131649115111Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854789120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103316367360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3974422528, + "inactive_split_bytes_peak": 5872025600, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:49.114711Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854789120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103316367360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3976519680, + "inactive_split_bytes_peak": 5872025600, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:48.253721Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3974422528, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 5872025600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104383643648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104383643648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 587, + "segment_count_current_delta": 0, + "segment_count_peak": 587, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T131649115111Z.pickle", + "timestamp_utc": "2026-08-01T13:16:49.216154Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T131649115111Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T131649115111Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8b733f609d3b147e9b2c5cb9d2e56eb9e20b207a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T131649115111Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24421dd876146d05744b46467bed5f05ffff091157c0df966cea59685c174a4c +size 5617610 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T131650099637Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T131650099637Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a958eef298952dea3567dc2a386552bec06987b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T131650099637Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854789120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103316367360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3980713984, + "inactive_split_bytes_peak": 5872025600, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:50.099236Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854789120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103316367360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3974422528, + "inactive_split_bytes_peak": 5872025600, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:49.230536Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3980713984, + "inactive_split_bytes_current_delta": 6291456, + "inactive_split_bytes_peak": 5872025600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104383643648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104383643648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 587, + "segment_count_current_delta": 0, + "segment_count_peak": 587, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T131650099637Z.pickle", + "timestamp_utc": "2026-08-01T13:16:50.198487Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T131650099637Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T131650099637Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..896fd59dc6fabd379bb9dc1440fe6b517b4ab861 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T131650099637Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d6d923c00a4c1da0810de06e459ae5cfcc2d2de53128ca567029a4b55995460 +size 5616622 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T131651082053Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T131651082053Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1fadab0fffdcb7e325ba45776bc6c67fb49bec6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T131651082053Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854789120, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103316367360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3980713984, + "inactive_split_bytes_peak": 5872025600, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:51.081657Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854789120, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103316367360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3980713984, + "inactive_split_bytes_peak": 5872025600, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:50.212887Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3980713984, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5872025600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104383643648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104383643648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 587, + "segment_count_current_delta": 0, + "segment_count_peak": 587, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T131651082053Z.pickle", + "timestamp_utc": "2026-08-01T13:16:51.182041Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T131651082053Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T131651082053Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db992d8d990dd5e375a80bfb925f9c76cbc54079 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T131651082053Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34be0e959334a6e2d1228a0b1f7731b7167fac82dd48ef580f8b7e3455582444 +size 5619527 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T131749814271Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T131749814271Z.json new file mode 100644 index 0000000000000000000000000000000000000000..350c57515eec4740329691e363770ce4207c7ea1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T131749814271Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804146176, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103316367360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66804146176, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66804146176, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3014238208, + "inactive_split_bytes_peak": 5872025600, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:17:49.813789Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804146176, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103316367360, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854789120, + "active_bytes_peak": 103316367360, + "allocated_bytes_current": 66854789120, + "allocated_bytes_peak": 103316367360, + "device": 4, + "inactive_split_bytes_current": 3980713984, + "inactive_split_bytes_peak": 5872025600, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104383643648, + "reserved_bytes_peak": 104383643648, + "segment_count_current": 587, + "segment_count_peak": 587, + "timestamp_utc": "2026-08-01T13:16:51.196462Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3014238208, + "inactive_split_bytes_current_delta": -966475776, + "inactive_split_bytes_peak": 5872025600, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104383643648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104383643648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 587, + "segment_count_current_delta": 0, + "segment_count_peak": 587, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T131749814271Z.pickle", + "timestamp_utc": "2026-08-01T13:17:50.033539Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T131749814271Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T131749814271Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a08ce109950a8569c245d80b2b83630225d97a39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T131749814271Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3670b07c9c02564b3992b7b5dfe44ebe8dca2904cf46d8103d4031298e42813 +size 9606879 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T131642596934Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T131642596934Z.json new file mode 100644 index 0000000000000000000000000000000000000000..da59bc3271add8d0be990eef7c656776a650c105 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T131642596934Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 79676544000, + "active_bytes_peak_delta": 44210207744, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 79676544000, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 79676544000, + "device": 5, + "inactive_split_bytes_current": 4265069568, + "inactive_split_bytes_peak": 5124378624, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 94359257088, + "reserved_bytes_peak": 94359257088, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:16:42.596338Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 79676544000, + "allocated_bytes_peak_delta": 44210207744, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:15:34.555562Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4265069568, + "inactive_split_bytes_current_delta": 4132250624, + "inactive_split_bytes_peak": 5124378624, + "inactive_split_bytes_peak_delta": 4963992576, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 94359257088, + "reserved_bytes_current_delta": 55115251712, + "reserved_bytes_peak": 94359257088, + "reserved_bytes_peak_delta": 55115251712, + "segment_count_current": 522, + "segment_count_current_delta": 451, + "segment_count_peak": 522, + "segment_count_peak_delta": 451, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T131642596934Z.pickle", + "timestamp_utc": "2026-08-01T13:16:42.708519Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T131642596934Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T131642596934Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6137a7702110f34d3da88edd5490d265af9a7c87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T131642596934Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78adb698d03c35d95618a4e879d29045c3647aee401456d7e9aef617a5eecd37 +size 5380311 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T131643592434Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T131643592434Z.json new file mode 100644 index 0000000000000000000000000000000000000000..38e6b77e1a95305bde5082fcae352eb4ab87ba6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T131643592434Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 23645672448, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3201813504, + "inactive_split_bytes_peak": 5124378624, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:43.592023Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 23645672448, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 79676544000, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 79676544000, + "device": 5, + "inactive_split_bytes_current": 4265069568, + "inactive_split_bytes_peak": 5124378624, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 94359257088, + "reserved_bytes_peak": 94359257088, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:16:42.727825Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3201813504, + "inactive_split_bytes_current_delta": -1063256064, + "inactive_split_bytes_peak": 5124378624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 10049552384, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 10049552384, + "segment_count_current": 592, + "segment_count_current_delta": 70, + "segment_count_peak": 592, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T131643592434Z.pickle", + "timestamp_utc": "2026-08-01T13:16:43.704405Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T131643592434Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T131643592434Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7af8c4db99b367c242fed620a4209722fbd4c031 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T131643592434Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fce5206ffbe6b0b468c5577d9c08a29b704e498e12cdcc340c67f844f79f2b38 +size 5615056 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T131644592243Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T131644592243Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f59d8e45832bf173cc03f7cea79a1d3999aca6d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T131644592243Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3879193600, + "inactive_split_bytes_peak": 5124378624, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:44.591832Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3201813504, + "inactive_split_bytes_peak": 5124378624, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:43.719719Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3879193600, + "inactive_split_bytes_current_delta": 677380096, + "inactive_split_bytes_peak": 5124378624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T131644592243Z.pickle", + "timestamp_utc": "2026-08-01T13:16:44.702360Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T131644592243Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T131644592243Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a7a8cfe47f6df6c8c872c5b3df506a04fc3215fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T131644592243Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2931043ab0b8aa2c27e1289302c397850c164f25ebdc289e3c28430131ab17fe +size 5613304 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T131645587244Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T131645587244Z.json new file mode 100644 index 0000000000000000000000000000000000000000..622ccb60f940750c6f8f1149e14c815ea176caca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T131645587244Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3883387904, + "inactive_split_bytes_peak": 5124378624, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:45.586832Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3879193600, + "inactive_split_bytes_peak": 5124378624, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:44.719722Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3883387904, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 5124378624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T131645587244Z.pickle", + "timestamp_utc": "2026-08-01T13:16:45.690376Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T131645587244Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T131645587244Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fbe93203eb825856904961ab3a6d8a6b408b046a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T131645587244Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c7bdd8090049600b4e227fcb2710f8b699d26ea52801012b6a2f25861016a4d +size 5613614 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T131647146082Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T131647146082Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eb0e3fc817aa146864e40470ecbcd97c26a59e33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T131647146082Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3885485056, + "inactive_split_bytes_peak": 5124378624, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:47.145673Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3883387904, + "inactive_split_bytes_peak": 5124378624, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:45.705111Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3885485056, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 5124378624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T131647146082Z.pickle", + "timestamp_utc": "2026-08-01T13:16:47.253968Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T131647146082Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T131647146082Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..09aaf8ccc723b2c1435470cd769bc37488c96125 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T131647146082Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b2ec6eb3ca72479b84cecfeaa029d315770b64b4da273066bf31a4bf1e6c5e6 +size 5612643 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T131648137955Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T131648137955Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a26dd137c3bfdfc238a33a706333e493d878d9e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T131648137955Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3879193600, + "inactive_split_bytes_peak": 5124378624, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:48.137562Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3885485056, + "inactive_split_bytes_peak": 5124378624, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:47.268555Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3879193600, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 5124378624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T131648137955Z.pickle", + "timestamp_utc": "2026-08-01T13:16:48.243054Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T131648137955Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T131648137955Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..298f5167601043aeb97a22de636cf01b43f0b5e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T131648137955Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8982b662d69656af80e9719fcf7639ebf24bb4438e8dab2c1597f72827c603a8 +size 5612981 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T131649115134Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T131649115134Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9497c35efb644e591b456c652825cf96bb4c87d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T131649115134Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3883387904, + "inactive_split_bytes_peak": 5124378624, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:49.114728Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3879193600, + "inactive_split_bytes_peak": 5124378624, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:48.258009Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3883387904, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 5124378624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T131649115134Z.pickle", + "timestamp_utc": "2026-08-01T13:16:49.217608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T131649115134Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T131649115134Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5722492235fa454f80f4a348465df2f6c092461c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T131649115134Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d052c46220c54ca472641e82d563ea51358176a47128a5affb59a7ceda861db5 +size 5613930 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T131650099654Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T131650099654Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed4b4d1c76eebfe73118c154172f295ea4c8410a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T131650099654Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3881290752, + "inactive_split_bytes_peak": 5124378624, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:50.099252Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3883387904, + "inactive_split_bytes_peak": 5124378624, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:49.232392Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3881290752, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 5124378624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T131650099654Z.pickle", + "timestamp_utc": "2026-08-01T13:16:50.202950Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T131650099654Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T131650099654Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f50e1de730c9261ffc1f399b872da88f8c4b785 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T131650099654Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6649449b4cbf46edaaa46fb4d099f9283e5d9d62056693c1e13937c097eeb5a +size 5612458 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T131651082149Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T131651082149Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42a8e311be5d77a0ed830bb87ccd63c052468d23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T131651082149Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3881290752, + "inactive_split_bytes_peak": 5124378624, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:51.081670Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3881290752, + "inactive_split_bytes_peak": 5124378624, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:50.218879Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3881290752, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5124378624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T131651082149Z.pickle", + "timestamp_utc": "2026-08-01T13:16:51.184538Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T131651082149Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T131651082149Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cb80253ca3fe5f3def39508d60442e29844548a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T131651082149Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:283e3fc04e1e347328ff634b68121ddde52a4e13a36d48467e796bb7933857ef +size 5612507 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T131749814263Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T131749814263Z.json new file mode 100644 index 0000000000000000000000000000000000000000..59b7c2087db321b8a2a922932460b034b3a8a9a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T131749814263Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103322216448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3103558656, + "inactive_split_bytes_peak": 5124378624, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:17:49.813786Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103322216448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103322216448, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103322216448, + "device": 5, + "inactive_split_bytes_current": 3881290752, + "inactive_split_bytes_peak": 5124378624, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:51.199080Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3103558656, + "inactive_split_bytes_current_delta": -777732096, + "inactive_split_bytes_peak": 5124378624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T131749814263Z.pickle", + "timestamp_utc": "2026-08-01T13:17:50.024401Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T131749814263Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T131749814263Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fcaf6a4289699422ec5e9846adf2f2466896818f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T131749814263Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9105b6ab759e682f8b351922dc2e6864e525607eaacc5f1c26eb1d3ff00b3b08 +size 9769836 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T131642596956Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T131642596956Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8730f9b56365bd228d7054e7f6c8f3d551dcdf8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T131642596956Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 79676544000, + "active_bytes_peak_delta": 44210207744, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 79676544000, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 79676544000, + "device": 6, + "inactive_split_bytes_current": 4032285696, + "inactive_split_bytes_peak": 5766784000, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93419732992, + "reserved_bytes_peak": 93419732992, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:16:42.596363Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 79676544000, + "allocated_bytes_peak_delta": 44210207744, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:15:34.555542Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4032285696, + "inactive_split_bytes_current_delta": 3899466752, + "inactive_split_bytes_peak": 5766784000, + "inactive_split_bytes_peak_delta": 5606397952, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93419732992, + "reserved_bytes_current_delta": 54175727616, + "reserved_bytes_peak": 93419732992, + "reserved_bytes_peak_delta": 54175727616, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T131642596956Z.pickle", + "timestamp_utc": "2026-08-01T13:16:42.706188Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T131642596956Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T131642596956Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a1be68e6b467651162a822fa86f1d0bee12cd231 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T131642596956Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6d9e4550cb84328067891dd608fa944b002f291fa607b1b4b46a7b5c16762fe +size 5378918 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T131643592513Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T131643592513Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6f91244122bf2275b112ef1906bf83edc660920 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T131643592513Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 23643837440, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3556232192, + "inactive_split_bytes_peak": 5766784000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:43.592097Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 23643837440, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 79676544000, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 79676544000, + "device": 6, + "inactive_split_bytes_current": 4032285696, + "inactive_split_bytes_peak": 5766784000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93419732992, + "reserved_bytes_peak": 93419732992, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:16:42.723413Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3556232192, + "inactive_split_bytes_current_delta": -476053504, + "inactive_split_bytes_peak": 5766784000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 10989076480, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 10989076480, + "segment_count_current": 590, + "segment_count_current_delta": 78, + "segment_count_peak": 590, + "segment_count_peak_delta": 78, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T131643592513Z.pickle", + "timestamp_utc": "2026-08-01T13:16:43.704170Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T131643592513Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T131643592513Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba7376182d82012aa385402aba68bfdd2345d475 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T131643592513Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31a5c43340ebc3cdba83b7a66b6356a5b2dd9cf4795deaa5fd7dbcbcdae9f65b +size 5612936 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T131644592334Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T131644592334Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aef69d1fec8ed9034584a55f66a0360cd8f4ba0d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T131644592334Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3673672704, + "inactive_split_bytes_peak": 5766784000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:44.591914Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3556232192, + "inactive_split_bytes_peak": 5766784000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:43.717787Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3673672704, + "inactive_split_bytes_current_delta": 117440512, + "inactive_split_bytes_peak": 5766784000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T131644592334Z.pickle", + "timestamp_utc": "2026-08-01T13:16:44.697387Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T131644592334Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T131644592334Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..07d213fad911d9b8a46fe40f74cea4fb886d99a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T131644592334Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed1091d407076b2d5253c17b4475843e1078bd612d7626fcdf7b8687f0da36fc +size 5614356 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T131645587206Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T131645587206Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3148413ec5fbad85b1d68df2228427a41905828b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T131645587206Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_peak": 5766784000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:45.586807Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3673672704, + "inactive_split_bytes_peak": 5766784000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:44.710490Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 5766784000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T131645587206Z.pickle", + "timestamp_utc": "2026-08-01T13:16:45.687170Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T131645587206Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T131645587206Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29de23d95aa56970fa03f0eab3df3da6f70ead4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T131645587206Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:309382af3c25445f4a0ffb43f05d94f466fe8b5a2c73b4729162d781d314af91 +size 5615383 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T131647146131Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T131647146131Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a8c4fcd07546a3bc2d38b6e258657b6eccea7e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T131647146131Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3562523648, + "inactive_split_bytes_peak": 5766784000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:47.145712Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_peak": 5766784000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:45.699982Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3562523648, + "inactive_split_bytes_current_delta": -113246208, + "inactive_split_bytes_peak": 5766784000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T131647146131Z.pickle", + "timestamp_utc": "2026-08-01T13:16:47.254285Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T131647146131Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T131647146131Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..62851677a3a11502d0a8f604f4a0cdb0ce5042a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T131647146131Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c825ebbf0d6e361bfa92f68b3d0a09fbc94e3c3ad4df7c701200bacda02f981 +size 5615016 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T131648138030Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T131648138030Z.json new file mode 100644 index 0000000000000000000000000000000000000000..49f85cd5aa44614dfc8cdefca81260aab214e551 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T131648138030Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_peak": 5766784000, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:48.137598Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3562523648, + "inactive_split_bytes_peak": 5766784000, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:47.269850Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_current_delta": 113246208, + "inactive_split_bytes_peak": 5766784000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T131648138030Z.pickle", + "timestamp_utc": "2026-08-01T13:16:48.235791Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T131648138030Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T131648138030Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..613af87ff3140c92d72ea8b750f5c41c3e389617 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T131648138030Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ae12b400c315ea399b4573ca18dba976073e636d724d0991c46eef94f515313 +size 5614499 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T131649115167Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T131649115167Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b9cd81f6cae9cf911d197bab96a0fc755453f621 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T131649115167Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3556232192, + "inactive_split_bytes_peak": 5766784000, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:49.114748Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_peak": 5766784000, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:48.248321Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3556232192, + "inactive_split_bytes_current_delta": -119537664, + "inactive_split_bytes_peak": 5766784000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T131649115167Z.pickle", + "timestamp_utc": "2026-08-01T13:16:49.212754Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T131649115167Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T131649115167Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..451ae7f2e3d5d202183b8a7ea13e5db983b8318f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T131649115167Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:904876b0f270144054d6a269508a1645d547aa3c01cd7b1d9fac38562a39fb0d +size 5613441 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T131650099704Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T131650099704Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a39dc2a3ce4e84ce0012de7adbc8121b09f0a461 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T131650099704Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_peak": 5766784000, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:50.099262Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3556232192, + "inactive_split_bytes_peak": 5766784000, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:49.225422Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_current_delta": 119537664, + "inactive_split_bytes_peak": 5766784000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T131650099704Z.pickle", + "timestamp_utc": "2026-08-01T13:16:50.201404Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T131650099704Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T131650099704Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..478d3d421318d5ebd0ec718bbe6919f3d424c7ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T131650099704Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c92614f37332af7eb7c2c3eeaada4f56775e93eed0f03a661f44109a0127084 +size 5613087 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T131651082096Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T131651082096Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e80717208243702c4c8971be9796439583cc7fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T131651082096Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3556232192, + "inactive_split_bytes_peak": 5766784000, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:51.081675Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3675769856, + "inactive_split_bytes_peak": 5766784000, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:50.214074Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3556232192, + "inactive_split_bytes_current_delta": -119537664, + "inactive_split_bytes_peak": 5766784000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T131651082096Z.pickle", + "timestamp_utc": "2026-08-01T13:16:51.177666Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T131651082096Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T131651082096Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44692f0a05b24d71876478c74afb74e44ad868e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T131651082096Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:802ece37d3cd1c8e0421e7f419c79f077d804220cf5894560140630dea87f4a3 +size 5613225 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T131749814324Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T131749814324Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4b0301e875a5711605be1e640aba2b44a1bb355 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T131749814324Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103320381440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3105655808, + "inactive_split_bytes_peak": 5766784000, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:17:49.813773Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103320381440, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 103320381440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 103320381440, + "device": 6, + "inactive_split_bytes_current": 3556232192, + "inactive_split_bytes_peak": 5766784000, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 590, + "segment_count_peak": 590, + "timestamp_utc": "2026-08-01T13:16:51.189883Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3105655808, + "inactive_split_bytes_current_delta": -450576384, + "inactive_split_bytes_peak": 5766784000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 590, + "segment_count_current_delta": 0, + "segment_count_peak": 590, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T131749814324Z.pickle", + "timestamp_utc": "2026-08-01T13:17:50.020620Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T131749814324Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T131749814324Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..550f8045e7bba64d7c03e84cf836037fc2d6921d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T131749814324Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ebd68fd19d6a4859fba69d44ee8a62859d8fb40d0c135f879f96cd069507f6b +size 9754846 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T131642596785Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T131642596785Z.json new file mode 100644 index 0000000000000000000000000000000000000000..529b6941ae9598cbe1137ef188a990e210ec5fa1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T131642596785Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 31622094336, + "active_bytes_peak": 79676544000, + "active_bytes_peak_delta": 44210207744, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 79676544000, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 79676544000, + "device": 7, + "inactive_split_bytes_current": 4267166208, + "inactive_split_bytes_peak": 5126385664, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 94359257088, + "reserved_bytes_peak": 94359257088, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:16:42.596302Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 31622094336, + "allocated_bytes_peak": 79676544000, + "allocated_bytes_peak_delta": 44210207744, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:15:34.555710Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4267166208, + "inactive_split_bytes_current_delta": 4134347264, + "inactive_split_bytes_peak": 5126385664, + "inactive_split_bytes_peak_delta": 4965999616, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 94359257088, + "reserved_bytes_current_delta": 55115251712, + "reserved_bytes_peak": 94359257088, + "reserved_bytes_peak_delta": 55115251712, + "segment_count_current": 522, + "segment_count_current_delta": 451, + "segment_count_peak": 522, + "segment_count_peak_delta": 451, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T131642596785Z.pickle", + "timestamp_utc": "2026-08-01T13:16:42.705641Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T131642596785Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T131642596785Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c5601736e7ef62dfdca15c0309545299e132e132 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T131642596785Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab164d08bab1bece41b196e4225d22213549324f1693b9fb211987c2657d5474 +size 5381982 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T131643592419Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T131643592419Z.json new file mode 100644 index 0000000000000000000000000000000000000000..68f229b92a09664e2d9e45137451c9b97f53c8b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T131643592419Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216960, + "active_bytes_peak_delta": 23645672960, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3203910144, + "inactive_split_bytes_peak": 5126385664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:43.592012Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216960, + "allocated_bytes_peak_delta": 23645672960, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 79676544000, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 79676544000, + "device": 7, + "inactive_split_bytes_current": 4267166208, + "inactive_split_bytes_peak": 5126385664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 94359257088, + "reserved_bytes_peak": 94359257088, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:16:42.722621Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3203910144, + "inactive_split_bytes_current_delta": -1063256064, + "inactive_split_bytes_peak": 5126385664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 10049552384, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 10049552384, + "segment_count_current": 592, + "segment_count_current_delta": 70, + "segment_count_peak": 592, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T131643592419Z.pickle", + "timestamp_utc": "2026-08-01T13:16:43.702021Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T131643592419Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T131643592419Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1357c206040ca39f774147f04ad8274458442868 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T131643592419Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3560e1d0e342bf056fb3af394ae2ed572db303cb1bb173790f4d96cf85e2eeb4 +size 5614941 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T131644592207Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T131644592207Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b0b04ad3148198f290b27db5b3d966bb54055ba7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T131644592207Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3883387392, + "inactive_split_bytes_peak": 5126385664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:44.591811Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216960, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3203910144, + "inactive_split_bytes_peak": 5126385664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:43.715061Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3883387392, + "inactive_split_bytes_current_delta": 679477248, + "inactive_split_bytes_peak": 5126385664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T131644592207Z.pickle", + "timestamp_utc": "2026-08-01T13:16:44.693135Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T131644592207Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T131644592207Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80cdc47cea5b62595d48c133f9ddd6b8252ab76a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T131644592207Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a90dcad34d3c2d4b8cae49103192033b78b8d5b8159afc86c9de490d8e83fef3 +size 5615569 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T131645587263Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T131645587263Z.json new file mode 100644 index 0000000000000000000000000000000000000000..edfa196ef88291130a8b923469e32c0a84ff73ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T131645587263Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3885484544, + "inactive_split_bytes_peak": 5126385664, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:45.586842Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216960, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3883387392, + "inactive_split_bytes_peak": 5126385664, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:44.705793Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3885484544, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 5126385664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T131645587263Z.pickle", + "timestamp_utc": "2026-08-01T13:16:45.687916Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T131645587263Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T131645587263Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a35da15584e2356105c4b8970ffbb19cca1e6e3d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T131645587263Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ee5d722f74c71ad10bdced92e770e046f22fd1dee4662863e4cb4634a220c35 +size 5616936 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T131647146154Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T131647146154Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0dcf38a9f1133884d32fa00bc90a32ddbf633f1b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T131647146154Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3881290240, + "inactive_split_bytes_peak": 5126385664, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:47.145724Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216960, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3885484544, + "inactive_split_bytes_peak": 5126385664, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:45.700576Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3881290240, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 5126385664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T131647146154Z.pickle", + "timestamp_utc": "2026-08-01T13:16:47.249692Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T131647146154Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T131647146154Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..95d57d0cd015e15b076ed85a4e32168ca46f0314 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T131647146154Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f90524e0e30ebdb1ac75fa49544d3ddb9c454747d5a219466fc2403aa56ba3e2 +size 5617172 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T131648137985Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T131648137985Z.json new file mode 100644 index 0000000000000000000000000000000000000000..473dcd1c520361a7e743f3cbcba992f717a202e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T131648137985Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3883387392, + "inactive_split_bytes_peak": 5126385664, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:48.137590Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216960, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3881290240, + "inactive_split_bytes_peak": 5126385664, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:47.262423Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3883387392, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 5126385664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T131648137985Z.pickle", + "timestamp_utc": "2026-08-01T13:16:48.235854Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T131648137985Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T131648137985Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..79d01b72c943ac9939af7ba93f3668f83456ce0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T131648137985Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec5462a1eb7db897ee934b4688503d9637b60fa15895dfe0c37727883c948619 +size 5616837 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T131649115173Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T131649115173Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d3e354931b0a240910c320d662742b552151ea72 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T131649115173Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3881290240, + "inactive_split_bytes_peak": 5126385664, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:49.114766Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216960, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3883387392, + "inactive_split_bytes_peak": 5126385664, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:48.248268Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3881290240, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 5126385664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T131649115173Z.pickle", + "timestamp_utc": "2026-08-01T13:16:49.212475Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T131649115173Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T131649115173Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c6cdcf47f14bba8a1d227ce3f60d92808854c8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T131649115173Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:995bdf19ee32b379e609ad299313d9a27e53ca926bd0bb83dbc51bf1a2c0db15 +size 5615667 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T131650099712Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T131650099712Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a1b5171a1e288c746b24fe17e506e2453122f900 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T131650099712Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3887581696, + "inactive_split_bytes_peak": 5126385664, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:50.099293Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216960, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3881290240, + "inactive_split_bytes_peak": 5126385664, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:49.224837Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3887581696, + "inactive_split_bytes_current_delta": 6291456, + "inactive_split_bytes_peak": 5126385664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T131650099712Z.pickle", + "timestamp_utc": "2026-08-01T13:16:50.200705Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T131650099712Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T131650099712Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc3cd266683e238f88c70edd8705eb3feb73ec1c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T131650099712Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a9086a43d030d2f79d397924acd56d567703254e0702caa2999045678c5d7d0 +size 5614131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T131651082130Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T131651082130Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e861da425a5b8df4b670d1bb4f52f9b9c77d9c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T131651082130Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103322216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3881290240, + "inactive_split_bytes_peak": 5126385664, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:51.081735Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103322216960, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3887581696, + "inactive_split_bytes_peak": 5126385664, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:50.213189Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3881290240, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 5126385664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T131651082130Z.pickle", + "timestamp_utc": "2026-08-01T13:16:51.181483Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T131651082130Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T131651082130Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29602f9df8edb356e6961830f162eca1877a5c22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T131651082130Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d875b45ba0f3bde344d7f0af96bcb323be32b15dc4f464ad724feafde4449236 +size 5615340 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T131749814265Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T131749814265Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d49683831707b9ce289bdd9e06d6fa07e68c53fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T131749814265Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906624, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103322216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906624, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66802906624, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3105655296, + "inactive_split_bytes_peak": 5126385664, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:17:49.813789Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906624, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103322216960, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 103322216960, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 103322216960, + "device": 7, + "inactive_split_bytes_current": 3881290240, + "inactive_split_bytes_peak": 5126385664, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104408809472, + "reserved_bytes_peak": 104408809472, + "segment_count_current": 592, + "segment_count_peak": 592, + "timestamp_utc": "2026-08-01T13:16:51.196298Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3105655296, + "inactive_split_bytes_current_delta": -775634944, + "inactive_split_bytes_peak": 5126385664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104408809472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104408809472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 592, + "segment_count_current_delta": 0, + "segment_count_peak": 592, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T131749814265Z.pickle", + "timestamp_utc": "2026-08-01T13:17:50.011583Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T131749814265Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T131749814265Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba2599c5519977011689e27fb5525bededfa8e38 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T131749814265Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b92dcc97c47fe3390d049fdc554620c79ba30c4a395afe1a7c7921dc50d7d83d +size 9771611 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..de4c9e32d4cac8e761b01fcfe64815b1ac540bfd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 2048 +capacity: 1.00 +expert_dispatch_overlap: off +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6025 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 226 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T132032647710Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T132032647710Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18b3ce9406e5556dc54d30c340ca51c9309dc781 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T132032647710Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 37208849408, + "active_bytes_peak": 79577833472, + "active_bytes_peak_delta": 44111497216, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 79577833472, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 79577833472, + "device": 0, + "inactive_split_bytes_current": 4483230720, + "inactive_split_bytes_peak": 7587278336, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 91555364864, + "reserved_bytes_peak": 91555364864, + "segment_count_current": 444, + "segment_count_peak": 444, + "timestamp_utc": "2026-08-01T13:20:32.647141Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 37208849408, + "allocated_bytes_peak": 79577833472, + "allocated_bytes_peak_delta": 44111497216, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:19:25.307619Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4483230720, + "inactive_split_bytes_current_delta": 4348314624, + "inactive_split_bytes_peak": 7587278336, + "inactive_split_bytes_peak_delta": 7426646528, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 91555364864, + "reserved_bytes_current_delta": 52309262336, + "reserved_bytes_peak": 91555364864, + "reserved_bytes_peak_delta": 52309262336, + "segment_count_current": 444, + "segment_count_current_delta": 372, + "segment_count_peak": 444, + "segment_count_peak_delta": 372, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T132032647710Z.pickle", + "timestamp_utc": "2026-08-01T13:20:32.765940Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T132032647710Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T132032647710Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..efc3f64f4bd0d3442433612d37e4c18bb4ededd2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T132032647710Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb50e5988513b8a49b5d6a9d785f14ae9fa1b80ee9487443f5667ff54b451e7a +size 5935533 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T132033775612Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T132033775612Z.json new file mode 100644 index 0000000000000000000000000000000000000000..015cf0c2d60b08fa75626d8e7d7d964af194a34b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T132033775612Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217196032, + "active_bytes_peak_delta": 23639362560, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4774734848, + "inactive_split_bytes_peak": 7587278336, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:33.775188Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217196032, + "allocated_bytes_peak_delta": 23639362560, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 79577833472, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 79577833472, + "device": 0, + "inactive_split_bytes_current": 4483230720, + "inactive_split_bytes_peak": 7587278336, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 91555364864, + "reserved_bytes_peak": 91555364864, + "segment_count_current": 444, + "segment_count_peak": 444, + "timestamp_utc": "2026-08-01T13:20:32.786449Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4774734848, + "inactive_split_bytes_current_delta": 291504128, + "inactive_split_bytes_peak": 7587278336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_current_delta": 12903776256, + "reserved_bytes_peak": 104459141120, + "reserved_bytes_peak_delta": 12903776256, + "segment_count_current": 530, + "segment_count_current_delta": 86, + "segment_count_peak": 530, + "segment_count_peak_delta": 86, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T132033775612Z.pickle", + "timestamp_utc": "2026-08-01T13:20:33.894978Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T132033775612Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T132033775612Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b63051fa8682f063a5c14d5afb592a96097ea3da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T132033775612Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f312942dbcefe0c2c22d17a37cf6dadf1ae213226a847665b56dd4f21d742456 +size 6638900 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T132034881749Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T132034881749Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2e6ce39252f617183941c8964f6accf08c495c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T132034881749Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217196032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4835552256, + "inactive_split_bytes_peak": 7587278336, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:34.881337Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217196032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4774734848, + "inactive_split_bytes_peak": 7587278336, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:33.912399Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4835552256, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 7587278336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104459141120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T132034881749Z.pickle", + "timestamp_utc": "2026-08-01T13:20:34.993216Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T132034881749Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T132034881749Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..139e775aea6545233a56f6c72bb810a5cc701c5a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T132034881749Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ec005939e8dcb0505b90386660a949a19fea625e1f0c25b0871de95a8ef4463 +size 6641556 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T132036263463Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T132036263463Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c734c92528f0b3f096ec4ece5f232c61b58ba567 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T132036263463Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217196032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4952992768, + "inactive_split_bytes_peak": 7587278336, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:36.263062Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217196032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4835552256, + "inactive_split_bytes_peak": 7587278336, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:35.010234Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4952992768, + "inactive_split_bytes_current_delta": 117440512, + "inactive_split_bytes_peak": 7587278336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104459141120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T132036263463Z.pickle", + "timestamp_utc": "2026-08-01T13:20:36.375584Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T132036263463Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T132036263463Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..514204ddf7025c4d6803ac072a67d113f9d113dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T132036263463Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:494ce8dc4c32c066f5c08c703b0de1524568a41ed4c5bb59cde8f1ec84655ac5 +size 6640523 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T132037657114Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T132037657114Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fbbb5735707bdf8086063ddf9ed9ef9240fa48c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T132037657114Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217196032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4600671232, + "inactive_split_bytes_peak": 7587278336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:37.656684Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217196032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4952992768, + "inactive_split_bytes_peak": 7587278336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:36.392169Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4600671232, + "inactive_split_bytes_current_delta": -352321536, + "inactive_split_bytes_peak": 7587278336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104459141120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T132037657114Z.pickle", + "timestamp_utc": "2026-08-01T13:20:37.768327Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T132037657114Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T132037657114Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..262cdd15273845db942c2d7292fceee5045141e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T132037657114Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:426e734f5df1ac8075303fab5963032c58cd122ee16d923dda95902d5436df17 +size 6640465 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T132038789934Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T132038789934Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9039fd9a692b0f739ef49314e6a0dbb65a0854aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T132038789934Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217196032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4952992768, + "inactive_split_bytes_peak": 7587278336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:38.789532Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217196032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4600671232, + "inactive_split_bytes_peak": 7587278336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:37.785114Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4952992768, + "inactive_split_bytes_current_delta": 352321536, + "inactive_split_bytes_peak": 7587278336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104459141120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T132038789934Z.pickle", + "timestamp_utc": "2026-08-01T13:20:38.900406Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T132038789934Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T132038789934Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2c7f9e26507de14581ed42b8192d6161f2ebad2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T132038789934Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:100e27f17df4d69477f4e5480fdad7440cd1db1402297c25b500e826cf0db129 +size 6640533 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T132039917136Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T132039917136Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e2ab28ed866f542ad4128576bd1b534503ea2aa8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T132039917136Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217196032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4600671232, + "inactive_split_bytes_peak": 7587278336, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:39.916712Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217196032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4952992768, + "inactive_split_bytes_peak": 7587278336, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:38.919448Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4600671232, + "inactive_split_bytes_current_delta": -352321536, + "inactive_split_bytes_peak": 7587278336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104459141120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T132039917136Z.pickle", + "timestamp_utc": "2026-08-01T13:20:40.030863Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T132039917136Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T132039917136Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8752e9b09f5c957501457e8365ef964f577ef481 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T132039917136Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1f2b56fffa8d6b242a9a2b5fcfa069092f2ae359a3467afcec47f8149dd14b8 +size 6640463 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T132041197957Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T132041197957Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04d26aebadfc07d176e6e09d0181e29b5a0a8458 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T132041197957Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217196032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4952992768, + "inactive_split_bytes_peak": 7587278336, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:41.197557Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217196032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4600671232, + "inactive_split_bytes_peak": 7587278336, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:40.047277Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4952992768, + "inactive_split_bytes_current_delta": 352321536, + "inactive_split_bytes_peak": 7587278336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104459141120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T132041197957Z.pickle", + "timestamp_utc": "2026-08-01T13:20:41.306278Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T132041197957Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T132041197957Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3b5418af4849b865198f86ed5c096199d3abf8c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T132041197957Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd397f9a54183afc77e3853887677c26f99de7d07ab7a0e8e1640a4eade11085 +size 6640530 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T132042329177Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T132042329177Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ed89e14daf03c0e22a7dbd5f83f4b99da241726 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T132042329177Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217196032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4600671232, + "inactive_split_bytes_peak": 7587278336, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:42.328764Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217196032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4952992768, + "inactive_split_bytes_peak": 7587278336, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:41.322926Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4600671232, + "inactive_split_bytes_current_delta": -352321536, + "inactive_split_bytes_peak": 7587278336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104459141120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T132042329177Z.pickle", + "timestamp_utc": "2026-08-01T13:20:42.436823Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T132042329177Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T132042329177Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5cf81ec5ff94664cc988dd1e4aa6d8b1932a053e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T132042329177Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fc7e38f36e650c82d6be7aa6a306e98365f1a2272a09c5c1f931c52f629d587 +size 6640527 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T132138579511Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T132138579511Z.json new file mode 100644 index 0000000000000000000000000000000000000000..24b0fad761f2af904ac2f08f11186855d55db186 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T132138579511Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166015488, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103217196032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166015488, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 61166015488, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4657294336, + "inactive_split_bytes_peak": 7587278336, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:21:38.579042Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166015488, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103217196032, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 103217196032, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 103217196032, + "device": 0, + "inactive_split_bytes_current": 4600671232, + "inactive_split_bytes_peak": 7587278336, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_peak": 104459141120, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T13:20:42.453295Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4657294336, + "inactive_split_bytes_current_delta": 56623104, + "inactive_split_bytes_peak": 7587278336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104459141120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104459141120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T132138579511Z.pickle", + "timestamp_utc": "2026-08-01T13:21:38.782693Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T132138579511Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T132138579511Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..40b954886cefc5b17a07bd04ff27a010f2777040 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T132138579511Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03ffc2c5701dd7dfb4bc623218d4ffa4a7edbd62d2bb723e3099b2d207a0c68f +size 10526224 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T132032647680Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T132032647680Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7db39933bb336c8939d0c84a33f06c04ad5119d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T132032647680Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 79575500800, + "active_bytes_peak_delta": 44109164544, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 79575500800, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 79575500800, + "device": 1, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_peak": 7589203456, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 91553267712, + "reserved_bytes_peak": 91553267712, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T13:20:32.647072Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 79575500800, + "allocated_bytes_peak_delta": 44109164544, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:19:25.308051Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_current_delta": 4346471424, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 7428817408, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 91553267712, + "reserved_bytes_current_delta": 52309262336, + "reserved_bytes_peak": 91553267712, + "reserved_bytes_peak_delta": 52309262336, + "segment_count_current": 443, + "segment_count_current_delta": 372, + "segment_count_peak": 443, + "segment_count_peak_delta": 372, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T132032647680Z.pickle", + "timestamp_utc": "2026-08-01T13:20:32.766333Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T132032647680Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T132032647680Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1fa52ae3ce4b7715b5b394699257f7b4371c769f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T132032647680Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ef1fac111279446a243ac738e84afd4d2e755f60711d29d6dccf492efdfedc2 +size 5926547 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T132033775594Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T132033775594Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d42099f2312770ca5777e55bc9338e81610b2ee9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T132033775594Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103205714944, + "active_bytes_peak_delta": 23630214144, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103205714944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103205714944, + "device": 1, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_peak": 7589203456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:33.775177Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205714944, + "allocated_bytes_peak_delta": 23630214144, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 79575500800, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 79575500800, + "device": 1, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_peak": 7589203456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 91553267712, + "reserved_bytes_peak": 91553267712, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T13:20:32.789149Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_current_delta": 291504128, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 12872318976, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 12872318976, + "segment_count_current": 525, + "segment_count_current_delta": 82, + "segment_count_peak": 525, + "segment_count_peak_delta": 82, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T132033775594Z.pickle", + "timestamp_utc": "2026-08-01T13:20:33.894706Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T132033775594Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T132033775594Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3a36916fad8cbf6a3f32b60db604709d90a35b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T132033775594Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:878b737613b5febbdb6824d127af761581b1f60e4ab400f64d65b89ae4589d48 +size 5947680 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T132034881749Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T132034881749Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fa5640a3665d5a9fc14b593a61e5576883a2224d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T132034881749Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 14134272, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_peak": 7589203456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:34.881341Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 14134272, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103205714944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103205714944, + "device": 1, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_peak": 7589203456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:33.912020Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T132034881749Z.pickle", + "timestamp_utc": "2026-08-01T13:20:34.994354Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T132034881749Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T132034881749Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1e9c10445b0d94f58e7fbf942fa462c7f487d003 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T132034881749Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02cb2045b0de3387d0a190a2cd955dd9319601ca857002009109091aabdd12d1 +size 5964353 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T132036263470Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T132036263470Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5cdf5c5ba20b8ee370b531a548de456188cfdc0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T132036263470Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:36.263047Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_peak": 7589203456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:35.011121Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T132036263470Z.pickle", + "timestamp_utc": "2026-08-01T13:20:36.376693Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T132036263470Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T132036263470Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3383cf5f1108f4ee129853ae92c54ac92ea0ec1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T132036263470Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beb90e24375f47f7305dce4f9f3d2f6c58ddba6a297d8127a8a7faddef2f5334 +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T132037657098Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T132037657098Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2f4066d5b0decdbc19ab7f82d8812daa9137e6c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T132037657098Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:37.656688Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:36.393772Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T132037657098Z.pickle", + "timestamp_utc": "2026-08-01T13:20:37.768530Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T132037657098Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T132037657098Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..68c15ecd986672da9d01dc21fc6e84c20732b0d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T132037657098Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8f9ae42f03d48b70f4181864b82ef2702050706f25e6d0fd35340a07dae7e97 +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T132038789962Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T132038789962Z.json new file mode 100644 index 0000000000000000000000000000000000000000..967723b6f69333cdb4c3b8c8c6494dc64de430f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T132038789962Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:38.789543Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:37.785315Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T132038789962Z.pickle", + "timestamp_utc": "2026-08-01T13:20:38.901081Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T132038789962Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T132038789962Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e07b62303bca382bf549c9c429ba7b5c4b0e630 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T132038789962Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dca942a54f8e279c0303daaa89922afba283f1978ab7fbdd9bbf1e49d75e5317 +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T132039917146Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T132039917146Z.json new file mode 100644 index 0000000000000000000000000000000000000000..798e060ed23d9d2ce9d897baf9c5366de5f8fc99 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T132039917146Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:39.916740Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:38.917693Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T132039917146Z.pickle", + "timestamp_utc": "2026-08-01T13:20:40.027365Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T132039917146Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T132039917146Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..19d9922f2784a5392626924e0b5499a32cc91705 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T132039917146Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2479e3be46a3196ff46bd29bff3bd5f3d30a1bbb46837b536a90f490fdae36a +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T132041197949Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T132041197949Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27ab22ece6c9f24bd0b68e0eb5276f938b8d2f1d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T132041197949Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:41.197547Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:40.043973Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T132041197949Z.pickle", + "timestamp_utc": "2026-08-01T13:20:41.307425Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T132041197949Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T132041197949Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a72e5636cafceae9d79f559d90633b1fd9fbbe4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T132041197949Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ffdb6970018a3117f48d944aed44abae7bc8c1e1820fe0632a63791bae8194e +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T132042329182Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T132042329182Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f9b392462098b9e490ef6d15650178743b3245d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T132042329182Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:42.328773Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:41.324156Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T132042329182Z.pickle", + "timestamp_utc": "2026-08-01T13:20:42.437117Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T132042329182Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T132042329182Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5bf4128f65c464e630b025dc371e5cf17dec3fd1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T132042329182Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eae8fc54dfcc3f0eb57c9e461ade8b0ec7a98453449d638c8949825d14dbe994 +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T132138579521Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T132138579521Z.json new file mode 100644 index 0000000000000000000000000000000000000000..17a42125f7f268200647c4257b994fa78001da6e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T132138579521Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 4653353984, + "inactive_split_bytes_peak": 7589203456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:21:38.579048Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 1, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:42.453505Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4653353984, + "inactive_split_bytes_current_delta": -356515840, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T132138579521Z.pickle", + "timestamp_utc": "2026-08-01T13:21:38.782534Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T132138579521Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T132138579521Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..06f1a8eb49a8fd7db7056a445471221497982400 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T132138579521Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46fde0e96f1a204892ecfae034233ac131148de56f82769b5e3488a3d002e245 +size 9725736 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T132032647587Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T132032647587Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e6d673e2f5c5f4ed18bd5e42a1dd534f7d6be5e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T132032647587Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 79575500800, + "active_bytes_peak_delta": 44109164544, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 79575500800, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 79575500800, + "device": 2, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_peak": 7589203456, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 91553267712, + "reserved_bytes_peak": 91553267712, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T13:20:32.646997Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 79575500800, + "allocated_bytes_peak_delta": 44109164544, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:19:25.307968Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_current_delta": 4346471424, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 7428817408, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 91553267712, + "reserved_bytes_current_delta": 52309262336, + "reserved_bytes_peak": 91553267712, + "reserved_bytes_peak_delta": 52309262336, + "segment_count_current": 443, + "segment_count_current_delta": 372, + "segment_count_peak": 443, + "segment_count_peak_delta": 372, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T132032647587Z.pickle", + "timestamp_utc": "2026-08-01T13:20:32.762128Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T132032647587Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T132032647587Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d2c7d7261f30f70609e027ffb2d08a967d0c99c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T132032647587Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b6803eb70fa21be5e1808239fe5fac7837702eaaa3f9a832c43213cfe07b783 +size 5926543 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T132033775659Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T132033775659Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5b5c31411658c893bcd0042a5d57adf35ac927b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T132033775659Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103205714944, + "active_bytes_peak_delta": 23630214144, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103205714944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103205714944, + "device": 2, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_peak": 7589203456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:33.775247Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205714944, + "allocated_bytes_peak_delta": 23630214144, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 79575500800, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 79575500800, + "device": 2, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_peak": 7589203456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 91553267712, + "reserved_bytes_peak": 91553267712, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T13:20:32.779069Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_current_delta": 291504128, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 12872318976, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 12872318976, + "segment_count_current": 525, + "segment_count_current_delta": 82, + "segment_count_peak": 525, + "segment_count_peak_delta": 82, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T132033775659Z.pickle", + "timestamp_utc": "2026-08-01T13:20:33.890668Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T132033775659Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T132033775659Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca0a5b03d283828e7c70aad295472da181706bae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T132033775659Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45c1f15bf7c89eb81f3c13c511e16a0de66454fd91e85ba9f11b8f80c1572caa +size 5947679 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T132034881768Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T132034881768Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a16e37d559240bf112f022e5a117199241bbf98d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T132034881768Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 14134272, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_peak": 7589203456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:34.881369Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 14134272, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103205714944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103205714944, + "device": 2, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_peak": 7589203456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:33.905065Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T132034881768Z.pickle", + "timestamp_utc": "2026-08-01T13:20:34.987278Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T132034881768Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T132034881768Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..73bb7c3151a6995e1c3750af42b540e8fa0859d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T132034881768Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e64f1a0e0f127f118b012067ed845dcc49b0095ae3e3fc50a39d16711b372ad +size 5964323 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T132036263443Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T132036263443Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d6121442b8326ffec434d16c1e5c66b056dd511e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T132036263443Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:36.263037Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_peak": 7589203456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:35.001022Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T132036263443Z.pickle", + "timestamp_utc": "2026-08-01T13:20:36.369148Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T132036263443Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T132036263443Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0137caa7d779883c244635acd896fe089c672e08 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T132036263443Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d1da0a569a0e49d6d37c3db2fc7081bcb569e4d3cca0ca44e6db4eeb914a711 +size 5964081 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T132037657100Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T132037657100Z.json new file mode 100644 index 0000000000000000000000000000000000000000..20b60e7bb79c3576523586a81aca7a67947f6e50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T132037657100Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:37.656678Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:36.382903Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T132037657100Z.pickle", + "timestamp_utc": "2026-08-01T13:20:37.767825Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T132037657100Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T132037657100Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b061c8b25950639157d8664ecad2874cd834eab2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T132037657100Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63116b8802ff6c041003bc7ec6de779f2e6c69171648fc63c4de5d80a48de4dc +size 5964047 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T132038789936Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T132038789936Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f3ecb124dca73ca2fc58afd798c4cbfb2b00232 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T132038789936Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:38.789521Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:37.784429Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T132038789936Z.pickle", + "timestamp_utc": "2026-08-01T13:20:38.893886Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T132038789936Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T132038789936Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1b99382a30a516c3aef45457906f398146d61e94 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T132038789936Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33ba28164aace06a846eb47f8a64fad9db1e0d30336c8cda3b63eee04646737a +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T132039917196Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T132039917196Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5b911c66e34aaee2b52e66c076263ea60d3b1af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T132039917196Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:39.916746Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:38.907435Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T132039917196Z.pickle", + "timestamp_utc": "2026-08-01T13:20:40.022622Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T132039917196Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T132039917196Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44a541f09f0f02e0c446a980446d6c166346b6fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T132039917196Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44dacd6e077f275f7fc5bf83a74d2847ee1d5d5e1b7b50941a01f60facc56d69 +size 5964046 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T132041197949Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T132041197949Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d8044dcb42e74ff176be46cc6ced8bc9ec52a48 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T132041197949Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:41.197522Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:40.036090Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T132041197949Z.pickle", + "timestamp_utc": "2026-08-01T13:20:41.299935Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T132041197949Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T132041197949Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1ebbcc3981f978929423b7ab4443b7efa03a41d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T132041197949Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e652f3944787340a5c4936209d6efdc910a9273b34e3819cc00a3c7d066df26e +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T132042329168Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T132042329168Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ba03235a09661bdd7726891e15267d0b5b114cf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T132042329168Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:42.328758Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:41.313287Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T132042329168Z.pickle", + "timestamp_utc": "2026-08-01T13:20:42.431030Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T132042329168Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T132042329168Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..317730be3f713d59f2a01efa813f461ab9f7ee3f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T132042329168Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dcc8b02faf76fcd526f5cdd8f3059c924318c6aa884f62c033b09be9f9bf357 +size 5964047 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T132138579524Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T132138579524Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4fe819d0dea38b0e6ffb04315d043328339f1e4c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T132138579524Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 4653353984, + "inactive_split_bytes_peak": 7589203456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:21:38.579050Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 2, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:42.445684Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4653353984, + "inactive_split_bytes_current_delta": -356515840, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T132138579524Z.pickle", + "timestamp_utc": "2026-08-01T13:21:38.790286Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T132138579524Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T132138579524Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..58f2a3fbdb9ff8f7b49557509673c7e8bc443338 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T132138579524Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d3a5bbb228c7590612e1eb563d8c26864e33cfaa4e0b53e3543459248778b89 +size 9725737 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T132032647727Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T132032647727Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9dda26365fe447b1410c1f5fcab11a88c6744904 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T132032647727Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 79575500800, + "active_bytes_peak_delta": 44109164544, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 79575500800, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 79575500800, + "device": 3, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_peak": 7589203456, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 91553267712, + "reserved_bytes_peak": 91553267712, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T13:20:32.647102Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 79575500800, + "allocated_bytes_peak_delta": 44109164544, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:19:25.308098Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_current_delta": 4346471424, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 7428817408, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 91553267712, + "reserved_bytes_current_delta": 52309262336, + "reserved_bytes_peak": 91553267712, + "reserved_bytes_peak_delta": 52309262336, + "segment_count_current": 443, + "segment_count_current_delta": 372, + "segment_count_peak": 443, + "segment_count_peak_delta": 372, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T132032647727Z.pickle", + "timestamp_utc": "2026-08-01T13:20:32.767410Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T132032647727Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T132032647727Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..22ed70a7cbd5fe534022fdcaa32631c9b2a6fb99 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T132032647727Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:427488002140edf4b881a1373e5dbf6ed1472192ebb6174f563170951bb404d7 +size 5926552 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T132033775627Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T132033775627Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cabd1ce53c95bdd8da8c9b146e8328dd978463ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T132033775627Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103205714944, + "active_bytes_peak_delta": 23630214144, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103205714944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103205714944, + "device": 3, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_peak": 7589203456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:33.775206Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205714944, + "allocated_bytes_peak_delta": 23630214144, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 79575500800, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 79575500800, + "device": 3, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_peak": 7589203456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 91553267712, + "reserved_bytes_peak": 91553267712, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T13:20:32.787970Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_current_delta": 291504128, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 12872318976, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 12872318976, + "segment_count_current": 525, + "segment_count_current_delta": 82, + "segment_count_peak": 525, + "segment_count_peak_delta": 82, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T132033775627Z.pickle", + "timestamp_utc": "2026-08-01T13:20:33.895562Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T132033775627Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T132033775627Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..452a4edbeb6f4affafa205e4788752cf56664063 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T132033775627Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b77ccbc3b5c1ea8f071d979aab8c17ac329f688e8dcf0b6a6e07588b6de320fa +size 5947687 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T132034881852Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T132034881852Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1b5f81c20feb196d47860a1769d43e169546319d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T132034881852Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 14134272, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_peak": 7589203456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:34.881440Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 14134272, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103205714944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103205714944, + "device": 3, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_peak": 7589203456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:33.912982Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T132034881852Z.pickle", + "timestamp_utc": "2026-08-01T13:20:34.994493Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T132034881852Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T132034881852Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..128d6de3b7e5429b7a0a2e7278008cd6b4dfc178 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T132034881852Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6feef945c70e07dcce0b0187a172a49c0a48b5544d8cc4cb6b4a84ae9187b8b +size 5964326 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T132036263497Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T132036263497Z.json new file mode 100644 index 0000000000000000000000000000000000000000..505420d8cab925f0e4dfae4166c9ff4854a45e44 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T132036263497Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:36.263074Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_peak": 7589203456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:35.011343Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T132036263497Z.pickle", + "timestamp_utc": "2026-08-01T13:20:36.374735Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T132036263497Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T132036263497Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..83e63b28eaf476cf96f1549de91c64c34d19aac0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T132036263497Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:574eb779aab92137f37c82903a10bb42d2d98868354c31866f41806b9dff0937 +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T132037657114Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T132037657114Z.json new file mode 100644 index 0000000000000000000000000000000000000000..747820ead92eaa879c94da1bb228566ea34ec9ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T132037657114Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:37.656698Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:36.391490Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T132037657114Z.pickle", + "timestamp_utc": "2026-08-01T13:20:37.768175Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T132037657114Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T132037657114Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13a59808eca83f82d0fe944588e2068250bd43f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T132037657114Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ef980d579c8c96002993eeea696d3a1dc1092b6c0823cab9aa4a8130f4b5477 +size 5964047 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T132038789987Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T132038789987Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a851124e7589efc9a838b4e1742f918074115ccb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T132038789987Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:38.789557Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:37.784966Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T132038789987Z.pickle", + "timestamp_utc": "2026-08-01T13:20:38.899876Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T132038789987Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T132038789987Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0aab26b942ed5a38a023ebdfe5b9572e74529c3a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T132038789987Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4823ec75fe725a89fb03fdebe0683d3e30caa5f7f48575ff81fd464cdb65469 +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T132039917172Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T132039917172Z.json new file mode 100644 index 0000000000000000000000000000000000000000..504c6b2caf1fbc226b2896002d05696cdf0f1e7a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T132039917172Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:39.916741Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:38.916533Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T132039917172Z.pickle", + "timestamp_utc": "2026-08-01T13:20:40.025369Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T132039917172Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T132039917172Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f324999f07f411f4a9c55990d796cef70bc8c48d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T132039917172Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6821923d49dcecab8491ed26dc4eee830060fcb94ebad78657b9728b999d59db +size 5964047 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T132041197979Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T132041197979Z.json new file mode 100644 index 0000000000000000000000000000000000000000..50f0d9cb0cc521f5fddb422ffbd9d572a4e72284 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T132041197979Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:41.197569Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:40.042003Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T132041197979Z.pickle", + "timestamp_utc": "2026-08-01T13:20:41.305270Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T132041197979Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T132041197979Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f4e21a9babf4a0d2f70f7ecec49a4d6ecb4592c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T132041197979Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c73a626187ebc4d102f6ef7737eb577ebd1e7cc0292c0a3b4fdb5427f5f1984 +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T132042329211Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T132042329211Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6154b228abbb585dbc4c30da332ca8a561127b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T132042329211Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:42.328798Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:41.321705Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T132042329211Z.pickle", + "timestamp_utc": "2026-08-01T13:20:42.435764Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T132042329211Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T132042329211Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a7261c85afd95af4f94501f08f387d7565698cca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T132042329211Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cc00ef693b177a03e182b8ae3e9b0fe5b16318fffe2893d8d0f806db88db349 +size 5964040 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T132138579522Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T132138579522Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5552efc51736202b15f079bc52dfc80b24b9a38 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T132138579522Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 4653353984, + "inactive_split_bytes_peak": 7589203456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:21:38.579062Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 3, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:42.452109Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4653353984, + "inactive_split_bytes_current_delta": -356515840, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T132138579522Z.pickle", + "timestamp_utc": "2026-08-01T13:21:38.783341Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T132138579522Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T132138579522Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..15b780d501482ad6233675ca01617f16fba1a4d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T132138579522Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3682833e6b37a278f5a99504a50883c4544bf9a4a3094eca9d1108625c710f6 +size 9725736 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T132032647596Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T132032647596Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e8294680f1edd518fc6a8028784b36ca2028dcf6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T132032647596Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441905152, + "active_bytes_current_delta": 37210007552, + "active_bytes_peak": 79578744832, + "active_bytes_peak_delta": 44111966208, + "after": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 79578744832, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 79578744832, + "device": 4, + "inactive_split_bytes_current": 4712316928, + "inactive_split_bytes_peak": 7465885184, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 91788148736, + "reserved_bytes_peak": 91788148736, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T13:20:32.647012Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441905152, + "allocated_bytes_current_delta": 37210007552, + "allocated_bytes_peak": 79578744832, + "allocated_bytes_peak_delta": 44111966208, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:19:25.307774Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4712316928, + "inactive_split_bytes_current_delta": 4579940352, + "inactive_split_bytes_peak": 7465885184, + "inactive_split_bytes_peak_delta": 7312475648, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 91788148736, + "reserved_bytes_current_delta": 52544143360, + "reserved_bytes_peak": 91788148736, + "reserved_bytes_peak_delta": 52544143360, + "segment_count_current": 441, + "segment_count_current_delta": 370, + "segment_count_peak": 441, + "segment_count_peak_delta": 370, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T132032647596Z.pickle", + "timestamp_utc": "2026-08-01T13:20:32.768966Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T132032647596Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T132032647596Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da0158a434bb6bfdf890b8627e8a979a4c72a485 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T132032647596Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7187fd8ab364d3ade0acc048ff73e9977b0557b4f201d1c9fed5cf468dd44b51 +size 5934969 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T132033775541Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T132033775541Z.json new file mode 100644 index 0000000000000000000000000000000000000000..21414ee81da74d89edb863fe5516b5952101b065 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T132033775541Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441905152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103212149760, + "active_bytes_peak_delta": 23633404928, + "after": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103212149760, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103212149760, + "device": 4, + "inactive_split_bytes_current": 5358239744, + "inactive_split_bytes_peak": 7465885184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:33.775123Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441905152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103212149760, + "allocated_bytes_peak_delta": 23633404928, + "before": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 79578744832, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 79578744832, + "device": 4, + "inactive_split_bytes_current": 4712316928, + "inactive_split_bytes_peak": 7465885184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 91788148736, + "reserved_bytes_peak": 91788148736, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T13:20:32.789003Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5358239744, + "inactive_split_bytes_current_delta": 645922816, + "inactive_split_bytes_peak": 7465885184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104371060736, + "reserved_bytes_current_delta": 12582912000, + "reserved_bytes_peak": 104371060736, + "reserved_bytes_peak_delta": 12582912000, + "segment_count_current": 523, + "segment_count_current_delta": 82, + "segment_count_peak": 523, + "segment_count_peak_delta": 82, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T132033775541Z.pickle", + "timestamp_utc": "2026-08-01T13:20:33.895423Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T132033775541Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T132033775541Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f30b38ea6bba4fa282ce216567c3f1ff5c954ea9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T132033775541Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b83f8b3ce7145db65e74041a841c120f4e6e7cd8f7dafc965a0ffc99d33b15a +size 6086613 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T132034881696Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T132034881696Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3df0e88a9782672ae11425e222c945d947b3515d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T132034881696Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441905152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217523712, + "active_bytes_peak_delta": 5373952, + "after": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:34.881283Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441905152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217523712, + "allocated_bytes_peak_delta": 5373952, + "before": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103212149760, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103212149760, + "device": 4, + "inactive_split_bytes_current": 5358239744, + "inactive_split_bytes_peak": 7465885184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:33.912661Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 7465885184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104371060736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104371060736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T132034881696Z.pickle", + "timestamp_utc": "2026-08-01T13:20:34.995661Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T132034881696Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T132034881696Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ed017bcbd9d53276ad2a616db86412349c2ec2ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T132034881696Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29fe2c289c213633342baaa7cf78b64c2c87dd88e4bc62d19ca6ea3ef0aa054c +size 6627028 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T132036263416Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T132036263416Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bcef24bb117cea8f9c50d6b03052ca3a16fade1c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T132036263416Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441905152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217523712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:36.263008Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441905152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217523712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:35.012487Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7465885184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104371060736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104371060736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T132036263416Z.pickle", + "timestamp_utc": "2026-08-01T13:20:36.378810Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T132036263416Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T132036263416Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..456de9390db076a7fd27e8c12fbeab7151381f6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T132036263416Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b747563a4465ece584a9314a618e341405ec16d46b34aa4d162bd8de63c475f7 +size 6626998 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T132037657095Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T132037657095Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9b48c5acf94557b00af99e1db8b532a48b9fbb65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T132037657095Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441905152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217523712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:37.656655Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441905152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217523712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:36.395367Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7465885184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104371060736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104371060736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T132037657095Z.pickle", + "timestamp_utc": "2026-08-01T13:20:37.771106Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T132037657095Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T132037657095Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..747990f9066df40c53855198b261ccd378cf48f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T132037657095Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca189835773db462b7150843248e88a489655226af2af6035dbdee0f2140ae81 +size 6627029 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T132038789960Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T132038789960Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c7289ac56ea5ba95fae3f6dbd7711b0ece3a6d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T132038789960Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441905152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217523712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:38.789506Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441905152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217523712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:37.787622Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7465885184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104371060736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104371060736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T132038789960Z.pickle", + "timestamp_utc": "2026-08-01T13:20:38.903622Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T132038789960Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T132038789960Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..afd1dd118b2e9aa09c3957d91ed198b98b32f145 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T132038789960Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0080c0a6c51fc0a9cd0f63aac00c584c64194214f713a08d07fbb6b35ba27064 +size 6626998 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T132039917158Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T132039917158Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ce9e2eb12d323c3f433535e81c1ee3d752fd954 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T132039917158Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441905152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217523712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:39.916693Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441905152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217523712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:38.920335Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7465885184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104371060736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104371060736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T132039917158Z.pickle", + "timestamp_utc": "2026-08-01T13:20:40.030193Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T132039917158Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T132039917158Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..51f1c41bd75cf347374a8f8e929e3d35bde6cd9c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T132039917158Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:519dddd8417e9dff765a2166ac4b819ffe4281b36fbd5f54fd5e88ed83661ac2 +size 6627039 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T132041198026Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T132041198026Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b56de2f302e6780922beec3ee3a69b4579ba491b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T132041198026Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441905152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217523712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:41.197536Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441905152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217523712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:40.047021Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7465885184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104371060736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104371060736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T132041198026Z.pickle", + "timestamp_utc": "2026-08-01T13:20:41.312047Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T132041198026Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T132041198026Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ed476f465052977763e209be945b2e938e1eb4e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T132041198026Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21729b9c24d48c6e5cd3dd2bb68a9aa581927543286796d711bb921ffd9197cc +size 6627006 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T132042329269Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T132042329269Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1cf404312bd5083129980abef1da1ca0916efbf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T132042329269Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441905152, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103217523712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:42.328781Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441905152, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103217523712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:41.329017Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7465885184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104371060736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104371060736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T132042329269Z.pickle", + "timestamp_utc": "2026-08-01T13:20:42.444772Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T132042329269Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T132042329269Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..61f61652e4ecc3061b923017a7e57c6714eb4193 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T132042329269Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2684842037a71f6d8e8a81fd8f7563768bbca6479ac12848628cf7a1a500e126 +size 6627026 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T132138579538Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T132138579538Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c9de09254e4f4d6d54155a03d5b757785fb9a1b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T132138579538Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61167616000, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103217523712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61167616000, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 61167616000, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5297422336, + "inactive_split_bytes_peak": 7465885184, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:21:38.579066Z", + "world_size": 8 + }, + "allocated_bytes_current": 61167616000, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103217523712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441905152, + "active_bytes_peak": 103217523712, + "allocated_bytes_current": 72441905152, + "allocated_bytes_peak": 103217523712, + "device": 4, + "inactive_split_bytes_current": 5240799232, + "inactive_split_bytes_peak": 7465885184, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 104371060736, + "reserved_bytes_peak": 104371060736, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T13:20:42.461813Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5297422336, + "inactive_split_bytes_current_delta": 56623104, + "inactive_split_bytes_peak": 7465885184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 104371060736, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104371060736, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T132138579538Z.pickle", + "timestamp_utc": "2026-08-01T13:21:38.789637Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T132138579538Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T132138579538Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c5f9bc6a71d4f414f7d98574fc68c52dce23f56d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T132138579538Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33ef834479e8cb8cb0e1b912a52c66eac2d01e021bb67e952362f06ace200ad4 +size 9981505 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T132032647646Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T132032647646Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5120c5f76ef37c072d7bbd827523f400f8a74495 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T132032647646Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 79575500800, + "active_bytes_peak_delta": 44109164544, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 79575500800, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 79575500800, + "device": 5, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_peak": 7589203456, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 91553267712, + "reserved_bytes_peak": 91553267712, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T13:20:32.647049Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 79575500800, + "allocated_bytes_peak_delta": 44109164544, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:19:25.307961Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_current_delta": 4346471424, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 7428817408, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 91553267712, + "reserved_bytes_current_delta": 52309262336, + "reserved_bytes_peak": 91553267712, + "reserved_bytes_peak_delta": 52309262336, + "segment_count_current": 443, + "segment_count_current_delta": 372, + "segment_count_peak": 443, + "segment_count_peak_delta": 372, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T132032647646Z.pickle", + "timestamp_utc": "2026-08-01T13:20:32.762422Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T132032647646Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T132032647646Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..30bf4115ec5b486650c001fed2adbf6ea88c2a34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T132032647646Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62bea5f63e2659b4dab567f91481dd399ab69378c3f5b931324e393255d2ac68 +size 5926542 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T132033775585Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T132033775585Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1a3498b348548b0127c768a8fa2a879160073f10 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T132033775585Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103205714944, + "active_bytes_peak_delta": 23630214144, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103205714944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103205714944, + "device": 5, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_peak": 7589203456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:33.775174Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205714944, + "allocated_bytes_peak_delta": 23630214144, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 79575500800, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 79575500800, + "device": 5, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_peak": 7589203456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 91553267712, + "reserved_bytes_peak": 91553267712, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T13:20:32.779381Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_current_delta": 291504128, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 12872318976, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 12872318976, + "segment_count_current": 525, + "segment_count_current_delta": 82, + "segment_count_peak": 525, + "segment_count_peak_delta": 82, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T132033775585Z.pickle", + "timestamp_utc": "2026-08-01T13:20:33.892822Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T132033775585Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T132033775585Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..98aa4722ddd862afba725751ed735a4abc26992b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T132033775585Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5615126f4aa327f624cd0131c2da3892baa37180dd6a61fce792c6cd5e567e0 +size 5947685 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T132034881790Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T132034881790Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b2d0d391a1f94b45b9b68f70da9dfc4386013665 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T132034881790Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 14134272, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_peak": 7589203456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:34.881385Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 14134272, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103205714944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103205714944, + "device": 5, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_peak": 7589203456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:33.905785Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T132034881790Z.pickle", + "timestamp_utc": "2026-08-01T13:20:34.989705Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T132034881790Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T132034881790Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc364284666b6bbf7be80366c3448c989ed56141 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T132034881790Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3476e88e52afc55a3701b311edc0481b49f93aeb2af7bda6d16d8956f6a701d2 +size 5964325 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T132036263418Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T132036263418Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fbd0bc8f37947595e6c788495c782169e054b317 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T132036263418Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:36.263025Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_peak": 7589203456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:35.001837Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T132036263418Z.pickle", + "timestamp_utc": "2026-08-01T13:20:36.370076Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T132036263418Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T132036263418Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2653cbb97698d92238c3c4598655b0ecd888762a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T132036263418Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ad2126980dae2c51cda5ee8bcf28ef2d31890a4da07c21e44925a0c619ebca4 +size 5964080 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T132037657096Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T132037657096Z.json new file mode 100644 index 0000000000000000000000000000000000000000..33ef035f418001831d42e7af66ddf0b53b3e30f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T132037657096Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:37.656682Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:36.383883Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T132037657096Z.pickle", + "timestamp_utc": "2026-08-01T13:20:37.766929Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T132037657096Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T132037657096Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..78410fa87ce26e7aabceb38be8ec2c8cce4195c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T132037657096Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a16f937ef4fcc4c4e75162cd4e75555d06f1b092a5090fd8d4e9d3f9f0ab6b0 +size 5964047 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T132038789976Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T132038789976Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b07ca34b59124d5e0435eb58769767c63db29c18 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T132038789976Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:38.789546Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:37.779301Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T132038789976Z.pickle", + "timestamp_utc": "2026-08-01T13:20:38.892693Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T132038789976Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T132038789976Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..41f0b0e5e01226d00cbac32a843a1c7f2a392448 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T132038789976Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e07600089f4df9f54d467b1ace372f1f0b30b874eca10110ce0b9c1bb9bd6875 +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T132039917154Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T132039917154Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65c2ea062e60935c3aab44fffa978d07a3705a7c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T132039917154Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:39.916720Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:38.904572Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T132039917154Z.pickle", + "timestamp_utc": "2026-08-01T13:20:40.025301Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T132039917154Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T132039917154Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93710c581dd7e272e90508407699ee39733269d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T132039917154Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bc9624c01d71572dce4b448d428b47ba90cfb35a82181a7c187eeeb1ae96daf +size 5964047 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T132041197945Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T132041197945Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f7f3cee2055383e4e6e09e145a03cc83ff698e87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T132041197945Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:41.197534Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:40.037544Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T132041197945Z.pickle", + "timestamp_utc": "2026-08-01T13:20:41.302246Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T132041197945Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T132041197945Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d763889d441b69979ad235e0a99ef5ea5e230536 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T132041197945Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8807eae664099e644f18162e0d61047d8663a7d199a540e3f8d3f504661de992 +size 5964080 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T132042329148Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T132042329148Z.json new file mode 100644 index 0000000000000000000000000000000000000000..399b964678a5ff5cea74ece0dc846b14cb7487a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T132042329148Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:42.328754Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:41.314037Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T132042329148Z.pickle", + "timestamp_utc": "2026-08-01T13:20:42.429188Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T132042329148Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T132042329148Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7b8196874fdad42b05162a07f747ddbb12cf3904 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T132042329148Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62421f4a14f0491a8cc550163cab0e6a689afd95ad3475907f2370e60bcb7e99 +size 5964042 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T132138579527Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T132138579527Z.json new file mode 100644 index 0000000000000000000000000000000000000000..92d5e46ddfad0860368f134013d84b806f28c3d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T132138579527Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 4653353984, + "inactive_split_bytes_peak": 7589203456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:21:38.579060Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 5, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:42.440574Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4653353984, + "inactive_split_bytes_current_delta": -356515840, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T132138579527Z.pickle", + "timestamp_utc": "2026-08-01T13:21:38.781322Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T132138579527Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T132138579527Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bff29f7a339e1b7c9dfe06fbe2e98437fd34cdf8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T132138579527Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:741453fb2e9c06f121ef522793b015a1bdad7fcf117e239078caf39e5635e5c6 +size 9725731 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T132032647573Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T132032647573Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8a4f7f5e5e617d446a5605c037727084f5ad3325 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T132032647573Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 79575500800, + "active_bytes_peak_delta": 44109164544, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 79575500800, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 79575500800, + "device": 6, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_peak": 7589203456, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 91553267712, + "reserved_bytes_peak": 91553267712, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T13:20:32.646989Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 79575500800, + "allocated_bytes_peak_delta": 44109164544, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:19:25.307954Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_current_delta": 4346471424, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 7428817408, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 91553267712, + "reserved_bytes_current_delta": 52309262336, + "reserved_bytes_peak": 91553267712, + "reserved_bytes_peak_delta": 52309262336, + "segment_count_current": 443, + "segment_count_current_delta": 372, + "segment_count_peak": 443, + "segment_count_peak_delta": 372, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T132032647573Z.pickle", + "timestamp_utc": "2026-08-01T13:20:32.761978Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T132032647573Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T132032647573Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf38f0db7a77f811ee1165c6f4b4716a0090edd6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T132032647573Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a9a6d333fd5a80fce7c3a517f52f1927fd49c6912166f48788a779aaef50144 +size 5926543 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T132033775577Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T132033775577Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cfb4856677a1a6a3db313a0a3addd5fd3f3049c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T132033775577Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103205714944, + "active_bytes_peak_delta": 23630214144, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103205714944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103205714944, + "device": 6, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_peak": 7589203456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:33.775148Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205714944, + "allocated_bytes_peak_delta": 23630214144, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 79575500800, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 79575500800, + "device": 6, + "inactive_split_bytes_current": 4479290368, + "inactive_split_bytes_peak": 7589203456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 91553267712, + "reserved_bytes_peak": 91553267712, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T13:20:32.778705Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_current_delta": 291504128, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 12872318976, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 12872318976, + "segment_count_current": 525, + "segment_count_current_delta": 82, + "segment_count_peak": 525, + "segment_count_peak_delta": 82, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T132033775577Z.pickle", + "timestamp_utc": "2026-08-01T13:20:33.893145Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T132033775577Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T132033775577Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f83817728105a2dbf6b6738d619085bbe8f22f26 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T132033775577Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6f05e33078bf0aa2e57d54e0ce347c9f543cffa46ed28d39934a3f51342d964 +size 5947684 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T132034881691Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T132034881691Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2eda86d0d8fd454fa411e331640531394bd65cb2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T132034881691Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 14134272, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_peak": 7589203456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:34.881287Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 14134272, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103205714944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103205714944, + "device": 6, + "inactive_split_bytes_current": 4770794496, + "inactive_split_bytes_peak": 7589203456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:33.907208Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T132034881691Z.pickle", + "timestamp_utc": "2026-08-01T13:20:34.996237Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T132034881691Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T132034881691Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c66a9935ec1277863932d9b7ab5287a385d6352 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T132034881691Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ef06759da4aef63b323e17033b6fa0facbcdf5acfc007459b5aa1b3ec14df52 +size 5964321 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T132036263403Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T132036263403Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6fca0209bc7c207db03bfe2260230add05cb8a0a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T132036263403Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:36.263006Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 4831611904, + "inactive_split_bytes_peak": 7589203456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:35.013896Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T132036263403Z.pickle", + "timestamp_utc": "2026-08-01T13:20:36.373597Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T132036263403Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T132036263403Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7bbbf80f00366d0c5beda109d716692ebfd4c064 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T132036263403Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e88da8f730f746650ab7d83c0a7904eeabcfb7b508a65821bbc9f1efae829eeb +size 5964072 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T132037657097Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T132037657097Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3cf6cd60cb3e97b7cb301b8f15254d2790a9bea9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T132037657097Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:37.656655Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:36.387152Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T132037657097Z.pickle", + "timestamp_utc": "2026-08-01T13:20:37.765338Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T132037657097Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T132037657097Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6402c92f6b8f241e9165ff452f02c6570586a9fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T132037657097Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e675e338c5ac340b569f12430dbb4b9d765fdd25b7f6f7347d3671886bc9dec1 +size 5964046 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T132038789898Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T132038789898Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eea11d2394aefbdeda6c648d80c0e6e10b9aeed1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T132038789898Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:38.789494Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:37.778835Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T132038789898Z.pickle", + "timestamp_utc": "2026-08-01T13:20:38.893850Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T132038789898Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T132038789898Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..036be2408e27903d768b4ab6a68ee79bad88d901 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T132038789898Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d037ff9a1548a10abc49890b2dcf70628195aad02c50a9217cd8c943332d95 +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T132039917077Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T132039917077Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bfd152a2bf8f3fe339535d9e64b6f50e2cc873f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T132039917077Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:39.916670Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:38.906998Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T132039917077Z.pickle", + "timestamp_utc": "2026-08-01T13:20:40.021420Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T132039917077Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T132039917077Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..09ec4082130458188eef23ffe71be9a505a73523 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T132039917077Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9658d1f323493df7d949f5221b6f2c6b0f126f8805778b4685292c5ada3607e +size 5964047 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T132041197955Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T132041197955Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2818f44553dde5041024bc8a652cbf9fe1a6c102 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T132041197955Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:41.197530Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:40.034640Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T132041197955Z.pickle", + "timestamp_utc": "2026-08-01T13:20:41.298881Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T132041197955Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T132041197955Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..efd54230eda5f38713bda87629bbcd221f54a29e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T132041197955Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac6acdee4e01c989cf0deb81c4a5da5371ef6b6ba0ff13add140ee7b36668f00 +size 5964078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T132042329133Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T132042329133Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0a6f573f774557b7dea06a31bdaeeecf43015584 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T132042329133Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:42.328734Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:41.312088Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T132042329133Z.pickle", + "timestamp_utc": "2026-08-01T13:20:42.434080Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T132042329133Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T132042329133Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b02cbad59c14a946e8408a5bed7a3543b99dc406 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T132042329133Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36fdf53f570555f09a6b636c905a96888b047fe69ce7de62ef067c36cc5098c1 +size 5964047 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T132138579593Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T132138579593Z.json new file mode 100644 index 0000000000000000000000000000000000000000..39771e6f2c824793b9d50f22924873938cefa33a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T132138579593Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103219849216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 4653353984, + "inactive_split_bytes_peak": 7589203456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:21:38.579045Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103219849216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 103219849216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 103219849216, + "device": 6, + "inactive_split_bytes_current": 5009869824, + "inactive_split_bytes_peak": 7589203456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:42.448607Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4653353984, + "inactive_split_bytes_current_delta": -356515840, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T132138579593Z.pickle", + "timestamp_utc": "2026-08-01T13:21:38.783797Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T132138579593Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T132138579593Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4577ef2086d928e09fcc392af871e8d06d37ea75 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T132138579593Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebd61d09138ba77f338b1b73a6dc179af87eae3cde4f03961153d3a5726fb8be +size 9725736 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T132032647666Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T132032647666Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f79cccb6439297abc35a530e324cd08d1f2ec4f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T132032647666Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 37208595968, + "active_bytes_peak": 79575500800, + "active_bytes_peak_delta": 44109164544, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 79575500800, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 79575500800, + "device": 7, + "inactive_split_bytes_current": 4479289856, + "inactive_split_bytes_peak": 7589203456, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 91553267712, + "reserved_bytes_peak": 91553267712, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T13:20:32.647098Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 37208595968, + "allocated_bytes_peak": 79575500800, + "allocated_bytes_peak_delta": 44109164544, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:19:25.307968Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4479289856, + "inactive_split_bytes_current_delta": 4346470912, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 7428817408, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 91553267712, + "reserved_bytes_current_delta": 52309262336, + "reserved_bytes_peak": 91553267712, + "reserved_bytes_peak_delta": 52309262336, + "segment_count_current": 443, + "segment_count_current_delta": 372, + "segment_count_peak": 443, + "segment_count_peak_delta": 372, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T132032647666Z.pickle", + "timestamp_utc": "2026-08-01T13:20:32.762533Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T132032647666Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T132032647666Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fff825dcc4eec01c167f51de7d2b3ad782a1b6ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T132032647666Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ea4b6764987af54c28781aafbd85c7d0eb396d2357112d6d504ab1d23f6e498 +size 5928144 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T132033775581Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T132033775581Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6142f5357c4da97ddfd8480db31b44d37c0d0486 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T132033775581Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103205715456, + "active_bytes_peak_delta": 23630214656, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103205715456, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103205715456, + "device": 7, + "inactive_split_bytes_current": 4770793984, + "inactive_split_bytes_peak": 7589203456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:33.775175Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205715456, + "allocated_bytes_peak_delta": 23630214656, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 79575500800, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 79575500800, + "device": 7, + "inactive_split_bytes_current": 4479289856, + "inactive_split_bytes_peak": 7589203456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 91553267712, + "reserved_bytes_peak": 91553267712, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T13:20:32.777643Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4770793984, + "inactive_split_bytes_current_delta": 291504128, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 12872318976, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 12872318976, + "segment_count_current": 525, + "segment_count_current_delta": 82, + "segment_count_peak": 525, + "segment_count_peak_delta": 82, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T132033775581Z.pickle", + "timestamp_utc": "2026-08-01T13:20:33.893164Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T132033775581Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T132033775581Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bc1ca4e798e715636b025c384f286c91a151cd37 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T132033775581Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c3f0f4548838d82f565765d67e20680ebceef4b95d3204460c305d71314956b +size 5949425 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T132034881789Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T132034881789Z.json new file mode 100644 index 0000000000000000000000000000000000000000..653e94071410ca729bb40246a92901f76aa96592 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T132034881789Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849728, + "active_bytes_peak_delta": 14134272, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 4831611392, + "inactive_split_bytes_peak": 7589203456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:34.881386Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849728, + "allocated_bytes_peak_delta": 14134272, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103205715456, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103205715456, + "device": 7, + "inactive_split_bytes_current": 4770793984, + "inactive_split_bytes_peak": 7589203456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:33.906300Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4831611392, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T132034881789Z.pickle", + "timestamp_utc": "2026-08-01T13:20:34.991816Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T132034881789Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T132034881789Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8200c82173d0368e977b975ba039652d5b301949 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T132034881789Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a5a1c31c9fde7808efd198af8c4be7e73c3197856a8346edd45807ed49c7bd5 +size 5966108 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T132036263496Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T132036263496Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d0a3dd4fd571d7e22b5edc54c6599224f80e182 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T132036263496Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_peak": 7589203456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:36.263099Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 4831611392, + "inactive_split_bytes_peak": 7589203456, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:35.004436Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T132036263496Z.pickle", + "timestamp_utc": "2026-08-01T13:20:36.365667Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T132036263496Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T132036263496Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ea6760a4bbcd2bd279465dda9b21ab1cc3f85c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T132036263496Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:073c0d4a72ae8434839f3815494cbb852229935be220272e3cc4d2d57a531c67 +size 5965808 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T132037657097Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T132037657097Z.json new file mode 100644 index 0000000000000000000000000000000000000000..397942767253505512618ca71b3a37de8f5e02c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T132037657097Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_peak": 7589203456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:37.656710Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_peak": 7589203456, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:36.377295Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T132037657097Z.pickle", + "timestamp_utc": "2026-08-01T13:20:37.759109Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T132037657097Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T132037657097Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..124f2ff81381b77fad90e42c9e21214e88db90a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T132037657097Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d4051bc2d1c02cd8e71c10252db92649b41181ca60477849520b9205d0a1ee9 +size 5965833 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T132038789965Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T132038789965Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1cab242bf549be383533e7afcfac3e0802760dc7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T132038789965Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_peak": 7589203456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:38.789571Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_peak": 7589203456, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:37.770654Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T132038789965Z.pickle", + "timestamp_utc": "2026-08-01T13:20:38.891439Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T132038789965Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T132038789965Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e03aaaf82799e85985f2323097264583ad85011 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T132038789965Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10f8f5e64db781e29158f35946e0d2bf2a3b7386db6a7819cf01f271a530f110 +size 5965833 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T132039917209Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T132039917209Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e0d6bbdedc1c09fd217d38f314061c42e5532ef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T132039917209Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_peak": 7589203456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:39.916782Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_peak": 7589203456, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:38.903117Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T132039917209Z.pickle", + "timestamp_utc": "2026-08-01T13:20:40.025123Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T132039917209Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T132039917209Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb1ad6769f7ecad20abb5228dc303cd177105229 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T132039917209Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09248860d4ef91e430d1484413860c7a9117a6c9205a31e0d7e2c24ceaf7b813 +size 5965802 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T132041198030Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T132041198030Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67e04842a8e7e3ed4afd302a9efa1c83239a2fda --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T132041198030Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_peak": 7589203456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:41.197623Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_peak": 7589203456, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:40.037547Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T132041198030Z.pickle", + "timestamp_utc": "2026-08-01T13:20:41.305714Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T132041198030Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T132041198030Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2fb369a9006d5b67ed2e06b71d0a0c6408d3bd0b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T132041198030Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:374b9cde9b613dfe3cf24ea64304dba5b2cab0b97917f92ea6fee5f8ac59ad5d +size 5965835 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T132042329206Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T132042329206Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ca40c371a8a4675ba155e5cdaf268d65fd77a9c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T132042329206Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103219849728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_peak": 7589203456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:42.328803Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103219849728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_peak": 7589203456, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:41.317875Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T132042329206Z.pickle", + "timestamp_utc": "2026-08-01T13:20:42.429409Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T132042329206Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T132042329206Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..42973c15e0e15d8a5b2008ae27c3cc5ebea7b95f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T132042329206Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ac6a6206d11eb6b641a45e652dd99ac023963f8858250a8a79f98ed6365e806 +size 5965834 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T132138579541Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T132138579541Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dcae973ecafd0561f37caaecc999df8678a42446 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T132138579541Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165762048, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103219849728, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165762048, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 61165762048, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 4653353472, + "inactive_split_bytes_peak": 7589203456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:21:38.579098Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165762048, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103219849728, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 103219849728, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 103219849728, + "device": 7, + "inactive_split_bytes_current": 5009869312, + "inactive_split_bytes_peak": 7589203456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104425586688, + "reserved_bytes_peak": 104425586688, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T13:20:42.440926Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4653353472, + "inactive_split_bytes_current_delta": -356515840, + "inactive_split_bytes_peak": 7589203456, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104425586688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104425586688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T132138579541Z.pickle", + "timestamp_utc": "2026-08-01T13:21:38.780305Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T132138579541Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T132138579541Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eca520fb8bc884d51c4b16fe29d2c6de869a6124 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T132138579541Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80460ea81b5e19e7dd16ccb26e9fc1509114237c6c94be414f9824915ef7c54d +size 9727459 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..e0ebaa3fb9712d6ff06df1b4a74aada1dac1ff0c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 2048 +capacity: 1.00 +expert_dispatch_overlap: on +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6026 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 226 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T132425383307Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T132425383307Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd355a299d2647f788b18ec4dae93501f2049bfe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T132425383307Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66856145920, + "active_bytes_current_delta": 31624690688, + "active_bytes_peak": 79671639552, + "active_bytes_peak_delta": 44205303296, + "after": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 79671639552, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 79671639552, + "device": 0, + "inactive_split_bytes_current": 2989501440, + "inactive_split_bytes_peak": 3657249792, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 95785320448, + "reserved_bytes_peak": 95785320448, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-01T13:24:25.382899Z", + "world_size": 8 + }, + "allocated_bytes_current": 66856145920, + "allocated_bytes_current_delta": 31624690688, + "allocated_bytes_peak": 79671639552, + "allocated_bytes_peak_delta": 44205303296, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:23:15.731173Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2989501440, + "inactive_split_bytes_current_delta": 2854585344, + "inactive_split_bytes_peak": 3657249792, + "inactive_split_bytes_peak_delta": 3496617984, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 95785320448, + "reserved_bytes_current_delta": 56539217920, + "reserved_bytes_peak": 95785320448, + "reserved_bytes_peak_delta": 56539217920, + "segment_count_current": 551, + "segment_count_current_delta": 479, + "segment_count_peak": 551, + "segment_count_peak_delta": 479, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T132425383307Z.pickle", + "timestamp_utc": "2026-08-01T13:24:25.492026Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T132425383307Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T132425383307Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dba23ee563b003e9c0348c2f1999471fdce7040d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T132425383307Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc9c2cc2ec67b759ade3f3bc6ac9af0727592ed521376157149c23da740021ee +size 5401368 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T132426425034Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T132426425034Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6edf6cd61e8e0999f426e239285640c09dfdaaeb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T132426425034Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66856145920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103309124096, + "active_bytes_peak_delta": 23637484544, + "after": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 2999987200, + "inactive_split_bytes_peak": 3686338560, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:26.424627Z", + "world_size": 8 + }, + "allocated_bytes_current": 66856145920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103309124096, + "allocated_bytes_peak_delta": 23637484544, + "before": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 79671639552, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 79671639552, + "device": 0, + "inactive_split_bytes_current": 2989501440, + "inactive_split_bytes_peak": 3657249792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 95785320448, + "reserved_bytes_peak": 95785320448, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-01T13:24:25.508370Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2999987200, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 3686338560, + "inactive_split_bytes_peak_delta": 29088768, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_current_delta": 8858370048, + "reserved_bytes_peak": 104643690496, + "reserved_bytes_peak_delta": 8858370048, + "segment_count_current": 608, + "segment_count_current_delta": 57, + "segment_count_peak": 608, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T132426425034Z.pickle", + "timestamp_utc": "2026-08-01T13:24:26.537076Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T132426425034Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T132426425034Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff8a6d8ffbaa2e87d3c09e81dacb3a95eb78d06b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T132426425034Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62e95ae81f8dc809ce4c93a1e3ff613e95eb794d02952a28f653188badb1c20d +size 5684959 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T132427414212Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T132427414212Z.json new file mode 100644 index 0000000000000000000000000000000000000000..685a73b93a2259c66ca8d06eeac4b660730d3d53 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T132427414212Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66856145920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103309124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 3004181504, + "inactive_split_bytes_peak": 4268237824, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:27.413824Z", + "world_size": 8 + }, + "allocated_bytes_current": 66856145920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103309124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 2999987200, + "inactive_split_bytes_peak": 3686338560, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:26.549670Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3004181504, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 4268237824, + "inactive_split_bytes_peak_delta": 581899264, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104643690496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 608, + "segment_count_current_delta": 0, + "segment_count_peak": 608, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T132427414212Z.pickle", + "timestamp_utc": "2026-08-01T13:24:27.515832Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T132427414212Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T132427414212Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d5a8651b53b70d85db5f07bd769e362a8f6b9162 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T132427414212Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d17dc45658a5b677432483359ee719d5a44158572f18ae6863d47e6de075aa02 +size 5633024 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T132428415065Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T132428415065Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94c00e90a5f8beae52edac59b0316b7bab39336c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T132428415065Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66856145920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103309124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 2758814720, + "inactive_split_bytes_peak": 4268237824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:28.414687Z", + "world_size": 8 + }, + "allocated_bytes_current": 66856145920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103309124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 3004181504, + "inactive_split_bytes_peak": 4268237824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:27.529590Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2758814720, + "inactive_split_bytes_current_delta": -245366784, + "inactive_split_bytes_peak": 4268237824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104643690496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 608, + "segment_count_current_delta": 0, + "segment_count_peak": 608, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T132428415065Z.pickle", + "timestamp_utc": "2026-08-01T13:24:28.509900Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T132428415065Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T132428415065Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..081739fa2d027e58cdcd909440e6d8b3b0b9585e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T132428415065Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:694da493aa8adcc2edb5285103fce81822bcb89985bac45dce582ac5ca429461 +size 5639016 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T132429897073Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T132429897073Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a8ce8c3ed530ec561d7af606c9128d5b3dee209 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T132429897073Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66856145920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103309124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 2763009024, + "inactive_split_bytes_peak": 4268237824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:29.896661Z", + "world_size": 8 + }, + "allocated_bytes_current": 66856145920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103309124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 2758814720, + "inactive_split_bytes_peak": 4268237824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:28.521179Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2763009024, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 4268237824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104643690496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 608, + "segment_count_current_delta": 0, + "segment_count_peak": 608, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T132429897073Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.001024Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T132429897073Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T132429897073Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14c121503fc26150c36176a46fc0f97f5849fc07 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T132429897073Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7177422e116136d983ca47b957c677e62efab4f28121c235b4d0a77b605fcc52 +size 5632621 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T132430888006Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T132430888006Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dbef095a4e2c812420d40e25b40db77f978b5fd1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T132430888006Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66856145920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103309124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 3008375808, + "inactive_split_bytes_peak": 4268237824, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:30.887619Z", + "world_size": 8 + }, + "allocated_bytes_current": 66856145920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103309124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 2763009024, + "inactive_split_bytes_peak": 4268237824, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:30.012790Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3008375808, + "inactive_split_bytes_current_delta": 245366784, + "inactive_split_bytes_peak": 4268237824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104643690496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 608, + "segment_count_current_delta": 0, + "segment_count_peak": 608, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T132430888006Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.982995Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T132430888006Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T132430888006Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a265ba58d0cb15dd77a00f1e90c6b60ea97bce7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T132430888006Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06d321081924dc29610cc7f524e3cb185781628b66685d633fcd3182d3bd6661 +size 5639199 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T132431904953Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T132431904953Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4fe8be1c47894d8934fb34fc0a480fe9328b7fc3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T132431904953Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66856145920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103309124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 2997890048, + "inactive_split_bytes_peak": 4268237824, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:31.904573Z", + "world_size": 8 + }, + "allocated_bytes_current": 66856145920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103309124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 3008375808, + "inactive_split_bytes_peak": 4268237824, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:30.994120Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2997890048, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 4268237824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104643690496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 608, + "segment_count_current_delta": 0, + "segment_count_peak": 608, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T132431904953Z.pickle", + "timestamp_utc": "2026-08-01T13:24:32.000544Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T132431904953Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T132431904953Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..81f6c8fa12f5ade3cbb13c43ba12b51689cdd5b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T132431904953Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8cefdf2d6104adfea75ebde97807d40c9096ccc2278890a533f50a1cc99afdc +size 5632642 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T132432915094Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T132432915094Z.json new file mode 100644 index 0000000000000000000000000000000000000000..25b6021ae3427cf6ce1fc231497d1ae92dd7f01c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T132432915094Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66856145920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103309124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 3004181504, + "inactive_split_bytes_peak": 4268237824, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:32.914698Z", + "world_size": 8 + }, + "allocated_bytes_current": 66856145920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103309124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 2997890048, + "inactive_split_bytes_peak": 4268237824, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:32.012525Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3004181504, + "inactive_split_bytes_current_delta": 6291456, + "inactive_split_bytes_peak": 4268237824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104643690496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 608, + "segment_count_current_delta": 0, + "segment_count_peak": 608, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T132432915094Z.pickle", + "timestamp_utc": "2026-08-01T13:24:33.008992Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T132432915094Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T132432915094Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..408f2f69c6e7394b4c988972a6f801be031a88b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T132432915094Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2f4521807b75b7d54b0d1f04f0368d5be2dca923823a9666f9c729effb17260 +size 5637797 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T132433915157Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T132433915157Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f78d8efcac325b09b9e5d5dd93127d60b7471ff5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T132433915157Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66856145920, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103309124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 2765106176, + "inactive_split_bytes_peak": 4268237824, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:33.914751Z", + "world_size": 8 + }, + "allocated_bytes_current": 66856145920, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103309124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 3004181504, + "inactive_split_bytes_peak": 4268237824, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:33.019984Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2765106176, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 4268237824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104643690496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 608, + "segment_count_current_delta": 0, + "segment_count_peak": 608, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T132433915157Z.pickle", + "timestamp_utc": "2026-08-01T13:24:34.007080Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T132433915157Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T132433915157Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..293114e7732d0dc088b375d49c1e5e28221a96cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T132433915157Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3d015d2061b3796ef1e5194c7a64db4a8246990a08e3259162e6324f3e21055 +size 5632530 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T132526823562Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T132526823562Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4ce6bb7864e833a2760c6f29dbf8cf77895dd4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T132526823562Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66805502976, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103309124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66805502976, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66805502976, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 2492787712, + "inactive_split_bytes_peak": 4268237824, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:25:26.823094Z", + "world_size": 8 + }, + "allocated_bytes_current": 66805502976, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103309124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66856145920, + "active_bytes_peak": 103309124096, + "allocated_bytes_current": 66856145920, + "allocated_bytes_peak": 103309124096, + "device": 0, + "inactive_split_bytes_current": 2765106176, + "inactive_split_bytes_peak": 4268237824, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_peak": 104643690496, + "segment_count_current": 608, + "segment_count_peak": 608, + "timestamp_utc": "2026-08-01T13:24:34.017961Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2492787712, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 4268237824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 104643690496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104643690496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 608, + "segment_count_current_delta": 0, + "segment_count_peak": 608, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T132526823562Z.pickle", + "timestamp_utc": "2026-08-01T13:25:27.017654Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T132526823562Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T132526823562Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..002a6c2c5c0f9523650acc5edc35282c3eee7d76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T132526823562Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ff3fd1e2aea1dd472b488ad420a4bc4ce3ab23245a969b2ecae4ba88e1d2f19 +size 10228645 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T132425383267Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T132425383267Z.json new file mode 100644 index 0000000000000000000000000000000000000000..99f9f3872e4cc6b711402d47ddb2b7ea8bd658e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T132425383267Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 31623011328, + "active_bytes_peak": 79670515200, + "active_bytes_peak_delta": 44204178944, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 79670515200, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 79670515200, + "device": 1, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_peak": 3770086400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 95076483072, + "reserved_bytes_peak": 95076483072, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T13:24:25.382698Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 31623011328, + "allocated_bytes_peak": 79670515200, + "allocated_bytes_peak_delta": 44204178944, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:23:15.731738Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_current_delta": 2782864384, + "inactive_split_bytes_peak": 3770086400, + "inactive_split_bytes_peak_delta": 3609700352, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 95076483072, + "reserved_bytes_current_delta": 55832477696, + "reserved_bytes_peak": 95076483072, + "reserved_bytes_peak_delta": 55832477696, + "segment_count_current": 541, + "segment_count_current_delta": 470, + "segment_count_peak": 541, + "segment_count_peak_delta": 470, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T132425383267Z.pickle", + "timestamp_utc": "2026-08-01T13:24:25.493106Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T132425383267Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T132425383267Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..73ba3e22d0d7893ea9f7ba25e29bb2239a7a7905 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T132425383267Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4bf5fd6ab15f81cc6960b579b693df6aa220292064840446393ab86635a02f0 +size 5391173 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T132426425067Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T132426425067Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db5a468f2cd2670137409b06e660cc45d64a5ac6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T132426425067Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 23643919872, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2926169088, + "inactive_split_bytes_peak": 3880810496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:26.424669Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 23643919872, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 79670515200, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 79670515200, + "device": 1, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_peak": 3770086400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 95076483072, + "reserved_bytes_peak": 95076483072, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T13:24:25.512123Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2926169088, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 3880810496, + "inactive_split_bytes_peak_delta": 110724096, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 9558818816, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 9558818816, + "segment_count_current": 603, + "segment_count_current_delta": 62, + "segment_count_peak": 603, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T132426425067Z.pickle", + "timestamp_utc": "2026-08-01T13:24:26.536595Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T132426425067Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T132426425067Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d233abb23c5dfc26fd3e1cb3086c3c61663b44c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T132426425067Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cf24070f8c0a49c7be4e22d36b35bb41a7b17c4e93daf9075d1a1b4b4cc5aa0 +size 5676906 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T132427414325Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T132427414325Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7dc3fe2bcd3484ed956605a7c96febddfefe16ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T132427414325Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4103991808, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:27.413917Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2926169088, + "inactive_split_bytes_peak": 3880810496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:26.551834Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 4103991808, + "inactive_split_bytes_peak_delta": 223181312, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T132427414325Z.pickle", + "timestamp_utc": "2026-08-01T13:24:27.527744Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T132427414325Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T132427414325Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0927e41e5150c7496b74ea1458b3b6e70cdf2456 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T132427414325Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0756a73c29f605e851a5236eb872392aeddc092e79482d9e1f9e69144484770b +size 5632200 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T132428415147Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T132428415147Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6d8888a12759b78e2600ca6c3c60b47274418755 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T132428415147Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2684996608, + "inactive_split_bytes_peak": 4103991808, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:28.414747Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4103991808, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:27.545651Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2684996608, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 4103991808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T132428415147Z.pickle", + "timestamp_utc": "2026-08-01T13:24:28.518041Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T132428415147Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T132428415147Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..387f0fa6d01370f320b431b100e2b70e8a290c6c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T132428415147Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e57fe9bbf513cf2107308aed1d5a04cc2bd611b1101a1879bb277d0afa67d51 +size 5631708 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T132429897126Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T132429897126Z.json new file mode 100644 index 0000000000000000000000000000000000000000..077220909983d5991f744ecedea91f91111e50ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T132429897126Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4103991808, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:29.896712Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2684996608, + "inactive_split_bytes_peak": 4103991808, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:28.532995Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 4103991808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T132429897126Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.003365Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T132429897126Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T132429897126Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c0e8d9dad42252d2402cc4337c4bdc6ecc18f3a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T132429897126Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36ec390a3446ceff739f6bb275e2dff919e6c9c859e6cf9917f3328b3efe97e6 +size 5631686 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T132430888110Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T132430888110Z.json new file mode 100644 index 0000000000000000000000000000000000000000..75440eaa201eed4fe8b604247862e7b6014e09b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T132430888110Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_peak": 4103991808, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:30.887687Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4103991808, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:30.018217Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 4103991808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T132430888110Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.991152Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T132430888110Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T132430888110Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ce27daf148281b1f13864fadde3cbf2c5e9a5143 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T132430888110Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7e905bb84c39e69762a88ff56f4b263e4de73a85783d744283eafbd74bf8f2c +size 5631932 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T132431905633Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T132431905633Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef01b97d4844ef2bc756a605b1b385d21683fe76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T132431905633Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4103991808, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:31.905013Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_peak": 4103991808, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:31.005887Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 4103991808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T132431905633Z.pickle", + "timestamp_utc": "2026-08-01T13:24:32.019368Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T132431905633Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T132431905633Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f0dd13d3b3ec87ec522672980834dfa9dfb2c5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T132431905633Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f7a61f8b3fdd0c8cdebd193e9f927d6091b419d96dcbc6aab6e9f32b0498a8b +size 5631793 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T132432915231Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T132432915231Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31d1e952177e8796e55ee1cfde57388d18e4bfae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T132432915231Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4110021120, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:32.914753Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4103991808, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:32.034209Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_current_delta": 6291456, + "inactive_split_bytes_peak": 4110021120, + "inactive_split_bytes_peak_delta": 6029312, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T132432915231Z.pickle", + "timestamp_utc": "2026-08-01T13:24:33.021573Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T132432915231Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T132432915231Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..73a296d45b18977941ddede2ed2708e05d0d5683 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T132432915231Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6a37c1a9e5f3b8fddf13d469c714a566a37202d51056c526f4e75edfaab8b0f +size 5631762 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T132433915158Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T132433915158Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6cf5a3a5a89b46338b146493a26ed05de31c43cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T132433915158Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2926169088, + "inactive_split_bytes_peak": 4171350016, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:33.914750Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4110021120, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:33.036859Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2926169088, + "inactive_split_bytes_current_delta": 230686720, + "inactive_split_bytes_peak": 4171350016, + "inactive_split_bytes_peak_delta": 61328896, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T132433915158Z.pickle", + "timestamp_utc": "2026-08-01T13:24:34.021681Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T132433915158Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T132433915158Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3663446a04a9fa1749d76dcc7ce29bdb73679a79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T132433915158Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6acfd9a75139fab79f250a99bbc44e8c0a745e090f9168f450003432d708f9dd +size 5631791 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T132526823622Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T132526823622Z.json new file mode 100644 index 0000000000000000000000000000000000000000..522f40db3fe2eed06da4921a81aebcb47529b55e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T132526823622Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803823616, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803823616, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66803823616, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2418969600, + "inactive_split_bytes_peak": 4171350016, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:25:26.823097Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803823616, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 1, + "inactive_split_bytes_current": 2926169088, + "inactive_split_bytes_peak": 4171350016, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:34.038949Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2418969600, + "inactive_split_bytes_current_delta": -507199488, + "inactive_split_bytes_peak": 4171350016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T132526823622Z.pickle", + "timestamp_utc": "2026-08-01T13:25:27.020731Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T132526823622Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T132526823622Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..668b1ca07bae8c3b0877891e313b26d9474d3c8c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T132526823622Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7efbb71a48f9385cda6c710e08bc7928ca8d227af9a75ed6c512289f4018e1e5 +size 9674128 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T132425383329Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T132425383329Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e9e37cecbc685e58f795834d891bf6a6bf40ff7f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T132425383329Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 31623011328, + "active_bytes_peak": 79670515200, + "active_bytes_peak_delta": 44204178944, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 79670515200, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 79670515200, + "device": 2, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_peak": 3770086400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 95076483072, + "reserved_bytes_peak": 95076483072, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T13:24:25.382711Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 31623011328, + "allocated_bytes_peak": 79670515200, + "allocated_bytes_peak_delta": 44204178944, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:23:15.731638Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_current_delta": 2782864384, + "inactive_split_bytes_peak": 3770086400, + "inactive_split_bytes_peak_delta": 3609700352, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 95076483072, + "reserved_bytes_current_delta": 55832477696, + "reserved_bytes_peak": 95076483072, + "reserved_bytes_peak_delta": 55832477696, + "segment_count_current": 541, + "segment_count_current_delta": 470, + "segment_count_peak": 541, + "segment_count_peak_delta": 470, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T132425383329Z.pickle", + "timestamp_utc": "2026-08-01T13:24:25.491477Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T132425383329Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T132425383329Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f376ea82cc046fd59f99f70bcd1eab710ccdfe69 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T132425383329Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1358e2f80fa7ff771025e2a46dc6811b5cb268b3f465509ac5ee627d2a1af849 +size 5391177 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T132426425094Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T132426425094Z.json new file mode 100644 index 0000000000000000000000000000000000000000..46c1be97de4ba7341c0cc626d2d1851d3a4b6f3a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T132426425094Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 23643919872, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2926169088, + "inactive_split_bytes_peak": 3880810496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:26.424697Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 23643919872, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 79670515200, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 79670515200, + "device": 2, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_peak": 3770086400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 95076483072, + "reserved_bytes_peak": 95076483072, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T13:24:25.507873Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2926169088, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 3880810496, + "inactive_split_bytes_peak_delta": 110724096, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 9558818816, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 9558818816, + "segment_count_current": 603, + "segment_count_current_delta": 62, + "segment_count_peak": 603, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T132426425094Z.pickle", + "timestamp_utc": "2026-08-01T13:24:26.536081Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T132426425094Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T132426425094Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..302aaa6229f224d6c0b1f2536a1800bca6f5b248 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T132426425094Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c5416b27a3d4ae62929086180a25cfd925b938b03063553f86c84c9b31a5f72 +size 5676918 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T132427414223Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T132427414223Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ba8b99b7a3bdda73c6d6c6826941d6d4baa2d96d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T132427414223Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4075589632, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:27.413813Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2926169088, + "inactive_split_bytes_peak": 3880810496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:26.550262Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 4075589632, + "inactive_split_bytes_peak_delta": 194779136, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T132427414223Z.pickle", + "timestamp_utc": "2026-08-01T13:24:27.510510Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T132427414223Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T132427414223Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7be243ec4e86900791e75ec421b59d0e78825c1d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T132427414223Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ac713125814931f9043e8efb38b56e810242728cc8f2f53bdc7c3cd8fd78501 +size 5632238 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T132428415079Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T132428415079Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b69c834190b361a46e7109519dc9e8af7f064096 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T132428415079Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2919877632, + "inactive_split_bytes_peak": 4165058560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:28.414689Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4075589632, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:27.521842Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2919877632, + "inactive_split_bytes_current_delta": 224395264, + "inactive_split_bytes_peak": 4165058560, + "inactive_split_bytes_peak_delta": 89468928, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T132428415079Z.pickle", + "timestamp_utc": "2026-08-01T13:24:28.514578Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T132428415079Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T132428415079Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b7a4629671e14850b99c47ffe526589de1289b1c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T132428415079Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29bc703465e4f242a5c4cd4c5ea734de024e9386b9f252dc1bcdf9ba59eb4334 +size 5631714 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T132429897079Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T132429897079Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27c0a408fd8c7ab2adef7e6918855f19875822b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T132429897079Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4165058560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:29.896673Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2919877632, + "inactive_split_bytes_peak": 4165058560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:28.525651Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 4165058560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T132429897079Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.002137Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T132429897079Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T132429897079Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..74c7140cf9e149198ef4cfcaa562e35e79fdce22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T132429897079Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8f37a2ea40bbb9214876e86b9f8e24b12ff7207777f7e08774baa33b8ce28cb +size 5631719 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T132430888027Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T132430888027Z.json new file mode 100644 index 0000000000000000000000000000000000000000..83824b47ba509e39d5b883d1f17aab6d81102d97 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T132430888027Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_peak": 4165058560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:30.887628Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4165058560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:30.014396Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 4165058560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T132430888027Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.982587Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T132430888027Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T132430888027Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f6c4a968a6d080455b44f5250fbf23e8f6d3a5d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T132430888027Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61114689d44c88e7c3ad3f0dbf12a2d10b787b122e01d090874986daf7252f3f +size 5631963 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T132431904995Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T132431904995Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cfc68c9ecc09fddd772866b1de668a8adf7c81e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T132431904995Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2924071936, + "inactive_split_bytes_peak": 4169252864, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:31.904582Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_peak": 4165058560, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:30.993656Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2924071936, + "inactive_split_bytes_current_delta": 224395264, + "inactive_split_bytes_peak": 4169252864, + "inactive_split_bytes_peak_delta": 4194304, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T132431904995Z.pickle", + "timestamp_utc": "2026-08-01T13:24:31.999232Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T132431904995Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T132431904995Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a08941adcb2d298cc2aaea5ac55efbfe54867e5e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T132431904995Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b95445710a5686c5ef91177f5b7ace4386ba5fa6f660d7e261be16ca93a403d +size 5631869 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T132432915119Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T132432915119Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f8af99ef4a5b104213539a023e7849b4fbe3b891 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T132432915119Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2930363392, + "inactive_split_bytes_peak": 4175544320, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:32.914708Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2924071936, + "inactive_split_bytes_peak": 4169252864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:32.010626Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2930363392, + "inactive_split_bytes_current_delta": 6291456, + "inactive_split_bytes_peak": 4175544320, + "inactive_split_bytes_peak_delta": 6291456, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T132432915119Z.pickle", + "timestamp_utc": "2026-08-01T13:24:33.006642Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T132432915119Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T132432915119Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..579daeb514a82c9a7b8ae3c7e82626fa5afb0c68 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T132432915119Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:330d7e2d6cd1affbb228df3b44d2ae22a5e4ff960cf75e1fad33e597d3b4c346 +size 5631773 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T132433915115Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T132433915115Z.json new file mode 100644 index 0000000000000000000000000000000000000000..866144971f5bbb4048b5abf67b8d7fd6d6f9f0a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T132433915115Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_peak": 4175544320, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:33.914718Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2930363392, + "inactive_split_bytes_peak": 4175544320, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:33.017323Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 4175544320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T132433915115Z.pickle", + "timestamp_utc": "2026-08-01T13:24:34.006785Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T132433915115Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T132433915115Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ecb0d3c73932f25f821ae6926076eed0cf3d681b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T132433915115Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c4158a01cd260ea8ca510b760acac2fd979509d2cc7f19eb6247c97e0aa27d2 +size 5631735 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T132526823670Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T132526823670Z.json new file mode 100644 index 0000000000000000000000000000000000000000..13601529b004f65c35017b18e83e2a1dfddaef74 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T132526823670Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803823616, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803823616, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66803823616, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2418969600, + "inactive_split_bytes_peak": 4175544320, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:25:26.823107Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803823616, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 2, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_peak": 4175544320, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:34.017436Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2418969600, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 4175544320, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T132526823670Z.pickle", + "timestamp_utc": "2026-08-01T13:25:27.017715Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T132526823670Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T132526823670Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93e051f52f868b61f72ed54baa7ef9f2e18384cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T132526823670Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e597f03b29e6b3f5cbf5f48ac17df946efa72c641bac1c1e2243254e194dcd5 +size 9674112 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T132425383306Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T132425383306Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca626cbbed255f3621ef322d895995f16c01ad38 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T132425383306Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 31623011328, + "active_bytes_peak": 79671957504, + "active_bytes_peak_delta": 44205621248, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 79671957504, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 79671957504, + "device": 3, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_peak": 3471848448, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 96250888192, + "reserved_bytes_peak": 96250888192, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-01T13:24:25.382715Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 31623011328, + "allocated_bytes_peak": 79671957504, + "allocated_bytes_peak_delta": 44205621248, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:23:15.731763Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_current_delta": 2782864384, + "inactive_split_bytes_peak": 3471848448, + "inactive_split_bytes_peak_delta": 3311462400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 96250888192, + "reserved_bytes_current_delta": 57006882816, + "reserved_bytes_peak": 96250888192, + "reserved_bytes_peak_delta": 57006882816, + "segment_count_current": 551, + "segment_count_current_delta": 480, + "segment_count_peak": 551, + "segment_count_peak_delta": 480, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T132425383306Z.pickle", + "timestamp_utc": "2026-08-01T13:24:25.494764Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T132425383306Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T132425383306Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c2115d9959f39fb8bce7dd4c253311da894ffb97 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T132425383306Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c67d086e9eeff6d4141126e41533b9f285f016fd1924307ebe1c2ef119bff03 +size 5394367 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T132426425122Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T132426425122Z.json new file mode 100644 index 0000000000000000000000000000000000000000..05f7ee0d6936a0557111e07aa303e79299229d4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T132426425122Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 23643517440, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_peak": 3471848448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:26.424711Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 23643517440, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 79671957504, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 79671957504, + "device": 3, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_peak": 3471848448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 96250888192, + "reserved_bytes_peak": 96250888192, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-01T13:24:25.513886Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_current_delta": -224395264, + "inactive_split_bytes_peak": 3471848448, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 8854175744, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 8854175744, + "segment_count_current": 606, + "segment_count_current_delta": 55, + "segment_count_peak": 606, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T132426425122Z.pickle", + "timestamp_utc": "2026-08-01T13:24:26.539814Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T132426425122Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T132426425122Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..42656013d8aa5df6045a28a1ba8ece30f79a7ff3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T132426425122Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17c68e2c7603ab339cfcfe65b92b21fa53f827f9141f4fff533f55bbf52ac37e +size 5678328 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T132427414238Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T132427414238Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f8e89d708877e7cbc411df315690ebab499effd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T132427414238Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4367634944, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:27.413835Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_peak": 3471848448, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:26.557089Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 895786496, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 606, + "segment_count_current_delta": 0, + "segment_count_peak": 606, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T132427414238Z.pickle", + "timestamp_utc": "2026-08-01T13:24:27.518790Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T132427414238Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T132427414238Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..37c06e1a7e5128e08cc86d25bf77784ea66b5c65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T132427414238Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aebd39c20473f5105c33e3e9775b34eea15ac4076507042d1da207c6e5b95ba +size 5632479 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T132428415179Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T132428415179Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a1a32f2cbb9d18c88b2000582cf20cf24bf89c8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T132428415179Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2919877632, + "inactive_split_bytes_peak": 4367634944, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:28.414750Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4367634944, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:27.533722Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2919877632, + "inactive_split_bytes_current_delta": 224395264, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 606, + "segment_count_current_delta": 0, + "segment_count_peak": 606, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T132428415179Z.pickle", + "timestamp_utc": "2026-08-01T13:24:28.519621Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T132428415179Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T132428415179Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c870eb3f70ffe77407feea9f7449a462e49b6a9a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T132428415179Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9068a8d12e58ed5cfd0a8adae7d30d95bf34297146a45ef10a5b66bee8d834cd +size 5631861 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T132429897103Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T132429897103Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e21f762351d830cb04894a0b60197f35f430ef6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T132429897103Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4367634944, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:29.896693Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2919877632, + "inactive_split_bytes_peak": 4367634944, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:28.534472Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 606, + "segment_count_current_delta": 0, + "segment_count_peak": 606, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T132429897103Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.003192Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T132429897103Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T132429897103Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..65d03eb15bb35530b3df77577fa47210e70d0253 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T132429897103Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:829b976246c78bb27b48f6b0c894c7b8d7faa80f36ba51cfcb175adaedad69a6 +size 5631817 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T132430888066Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T132430888066Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76056f1a5399daf45d2fd40b4d489391f43bb3fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T132430888066Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_peak": 4367634944, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:30.887670Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4367634944, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:30.018138Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 606, + "segment_count_current_delta": 0, + "segment_count_peak": 606, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T132430888066Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.990918Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T132430888066Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T132430888066Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..85a506adeee2455aa478eccaa555fcd6ddbb2a50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T132430888066Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20505b49f821618687b7a47c705394e6b71bc5ce76b86a95d6e28f7407e9557b +size 5632040 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T132431905040Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T132431905040Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c490580199eb7c72702657333ddbc9fe874d8fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T132431905040Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4367634944, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:31.904632Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_peak": 4367634944, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:31.005634Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 606, + "segment_count_current_delta": 0, + "segment_count_peak": 606, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T132431905040Z.pickle", + "timestamp_utc": "2026-08-01T13:24:32.004435Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T132431905040Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T132431905040Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..22cc43ffaf88c0efeb97a8c318e6947f7902a66b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T132431905040Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc53df1302a1a32139cfa8937c5cd256bab6ce733350d0783808a7b5e763678c +size 5631865 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T132432915142Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T132432915142Z.json new file mode 100644 index 0000000000000000000000000000000000000000..138996c5bdc9243f5a4a107389317dd841be45e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T132432915142Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2930363392, + "inactive_split_bytes_peak": 4367634944, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:32.914737Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4367634944, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:32.017206Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2930363392, + "inactive_split_bytes_current_delta": 241172480, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 606, + "segment_count_current_delta": 0, + "segment_count_peak": 606, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T132432915142Z.pickle", + "timestamp_utc": "2026-08-01T13:24:33.015817Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T132432915142Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T132432915142Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fa8f726aa02db32945f5cd2247d1a02cacf6d2f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T132432915142Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0ae62f840dc21a7c0e9a6e9d64d1f571db3731b6bc031b039499c73ab577d8a +size 5631798 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T132433915261Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T132433915261Z.json new file mode 100644 index 0000000000000000000000000000000000000000..509ede77a08fa19ab0741f66b999569c75c5bea2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T132433915261Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_peak": 4382167552, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:33.914767Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2930363392, + "inactive_split_bytes_peak": 4367634944, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:33.030172Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 4382167552, + "inactive_split_bytes_peak_delta": 14532608, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 606, + "segment_count_current_delta": 0, + "segment_count_peak": 606, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T132433915261Z.pickle", + "timestamp_utc": "2026-08-01T13:24:34.016799Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T132433915261Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T132433915261Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b68a326c6cdc371dfe8e26b1286066972668a763 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T132433915261Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9105d6d53ebe137ac97deede41e49093e056be22697394a58cdbd9e0b6ec1487 +size 5631779 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T132526823675Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T132526823675Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d301c9f38b378aa883ffb4a683d775e21fc2a8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T132526823675Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803823616, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803823616, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66803823616, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2418969600, + "inactive_split_bytes_peak": 4382167552, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:25:26.823116Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803823616, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 3, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_peak": 4382167552, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 606, + "segment_count_peak": 606, + "timestamp_utc": "2026-08-01T13:24:34.031421Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2418969600, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 4382167552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 606, + "segment_count_current_delta": 0, + "segment_count_peak": 606, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T132526823675Z.pickle", + "timestamp_utc": "2026-08-01T13:25:27.019937Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T132526823675Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T132526823675Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1fc1fbcc9407b046618f5e9e712c77e4b5352f3b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T132526823675Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a0334e994ba56bd20552409306ddbbf975cd29a9ad781e1cc792a8d2dff674b +size 10221185 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T132425383211Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T132425383211Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1722d84a11e16047971cd0aeaf63061546ca3463 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T132425383211Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855763968, + "active_bytes_current_delta": 31623866368, + "active_bytes_peak": 79665735168, + "active_bytes_peak_delta": 44198956544, + "after": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 79665735168, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 79665735168, + "device": 4, + "inactive_split_bytes_current": 2918580224, + "inactive_split_bytes_peak": 3786629120, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 95548342272, + "reserved_bytes_peak": 95548342272, + "segment_count_current": 544, + "segment_count_peak": 544, + "timestamp_utc": "2026-08-01T13:24:25.382786Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855763968, + "allocated_bytes_current_delta": 31623866368, + "allocated_bytes_peak": 79665735168, + "allocated_bytes_peak_delta": 44198956544, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:23:15.731403Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2918580224, + "inactive_split_bytes_current_delta": 2786203648, + "inactive_split_bytes_peak": 3786629120, + "inactive_split_bytes_peak_delta": 3633219584, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 95548342272, + "reserved_bytes_current_delta": 56304336896, + "reserved_bytes_peak": 95548342272, + "reserved_bytes_peak_delta": 56304336896, + "segment_count_current": 544, + "segment_count_current_delta": 473, + "segment_count_peak": 544, + "segment_count_peak_delta": 473, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T132425383211Z.pickle", + "timestamp_utc": "2026-08-01T13:24:25.493899Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T132425383211Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T132425383211Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d90362a1d5e56f826159a64d65509dac33891537 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T132425383211Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a73f5dce0ce69be9de86a2844697c4ed51f397ffd4b819fab278d57b6c127554 +size 5400659 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T132426425040Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T132426425040Z.json new file mode 100644 index 0000000000000000000000000000000000000000..17489dc1b9da0d1be112b30f98954dc567602aa3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T132426425040Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855763968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103307860480, + "active_bytes_peak_delta": 23642125312, + "after": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2694184960, + "inactive_split_bytes_peak": 4140574208, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:26.424619Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855763968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103307860480, + "allocated_bytes_peak_delta": 23642125312, + "before": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 79665735168, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 79665735168, + "device": 4, + "inactive_split_bytes_current": 2918580224, + "inactive_split_bytes_peak": 3786629120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 95548342272, + "reserved_bytes_peak": 95548342272, + "segment_count_current": 544, + "segment_count_peak": 544, + "timestamp_utc": "2026-08-01T13:24:25.513321Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2694184960, + "inactive_split_bytes_current_delta": -224395264, + "inactive_split_bytes_peak": 4140574208, + "inactive_split_bytes_peak_delta": 353945088, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 105115549696, + "reserved_bytes_current_delta": 9567207424, + "reserved_bytes_peak": 105115549696, + "reserved_bytes_peak_delta": 9567207424, + "segment_count_current": 609, + "segment_count_current_delta": 65, + "segment_count_peak": 609, + "segment_count_peak_delta": 65, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T132426425040Z.pickle", + "timestamp_utc": "2026-08-01T13:24:26.538106Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T132426425040Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T132426425040Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d2415ceda1a3f48c16ec98758ee1f94a2beb684d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T132426425040Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0b134913ed98f367c77687a08d588752b751caed588db9fda61e046e59e3e08 +size 5684557 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T132427414224Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T132427414224Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fff314fb1dedf704d450c7b2552546fc72b81df2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T132427414224Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855763968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103307860480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2698379264, + "inactive_split_bytes_peak": 4140574208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:27.413805Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855763968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103307860480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2694184960, + "inactive_split_bytes_peak": 4140574208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:26.553290Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2698379264, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 4140574208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 105115549696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105115549696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 609, + "segment_count_current_delta": 0, + "segment_count_peak": 609, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T132427414224Z.pickle", + "timestamp_utc": "2026-08-01T13:24:27.518637Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T132427414224Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T132427414224Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a0df68917eaf4cab9965260c7319832bebe84c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T132427414224Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbd788a2251b357192edcd23a4c15a51d62bdfb2c5f4cb728d1d771aa3e1ebd5 +size 5637786 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T132428415101Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T132428415101Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27a41b31c7c7039336fb6ac0dbf5631df1f6b94c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T132428415101Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855763968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103307860480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2922774528, + "inactive_split_bytes_peak": 4140574208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:28.414699Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855763968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103307860480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2698379264, + "inactive_split_bytes_peak": 4140574208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:27.533487Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2922774528, + "inactive_split_bytes_current_delta": 224395264, + "inactive_split_bytes_peak": 4140574208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 105115549696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105115549696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 609, + "segment_count_current_delta": 0, + "segment_count_peak": 609, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T132428415101Z.pickle", + "timestamp_utc": "2026-08-01T13:24:28.519141Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T132428415101Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T132428415101Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e2d32c360bc9dd8a5d8ab4fa74e0257fd89eb4e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T132428415101Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63817237a7b03ca523779b2e8f28c5249bf0cffa8c7f3ca7ad57c52cfa002d6b +size 5638909 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T132429897106Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T132429897106Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d6268024f1a670ef37bcefb034cd8b6b8a060166 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T132429897106Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855763968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103307860480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2692087808, + "inactive_split_bytes_peak": 4140574208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:29.896664Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855763968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103307860480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2922774528, + "inactive_split_bytes_peak": 4140574208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:28.534085Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2692087808, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 4140574208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 105115549696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105115549696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 609, + "segment_count_current_delta": 0, + "segment_count_peak": 609, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T132429897106Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.003765Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T132429897106Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T132429897106Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6f7272a72824d9b5284a0b56755b88a4f08fddb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T132429897106Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7edce849cab5b362d06e27147de7e29787eb69b77f0dbfb2717c5ccf9be3e8c +size 5638846 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T132430888052Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T132430888052Z.json new file mode 100644 index 0000000000000000000000000000000000000000..388cf8fa1f11e8cb06fbf11e1218ed12afb81041 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T132430888052Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855763968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103307860480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2702573568, + "inactive_split_bytes_peak": 4140574208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:30.887642Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855763968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103307860480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2692087808, + "inactive_split_bytes_peak": 4140574208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:30.018551Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2702573568, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 4140574208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 105115549696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105115549696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 609, + "segment_count_current_delta": 0, + "segment_count_peak": 609, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T132430888052Z.pickle", + "timestamp_utc": "2026-08-01T13:24:31.001950Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T132430888052Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T132430888052Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..64214551a961112206f4a176046eadead4b5b589 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T132430888052Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9133e1d2a8df8bfa48d8beb473a374bc470217153d2f18201148bdd09aa53fa +size 5639090 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T132431904970Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T132431904970Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27410256e67c474af4ec7f47d8da3a9563ab0558 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T132431904970Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855763968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103307860480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2692087808, + "inactive_split_bytes_peak": 4140574208, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:31.904571Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855763968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103307860480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2702573568, + "inactive_split_bytes_peak": 4140574208, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:31.019294Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2692087808, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 4140574208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 105115549696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105115549696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 609, + "segment_count_current_delta": 0, + "segment_count_peak": 609, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T132431904970Z.pickle", + "timestamp_utc": "2026-08-01T13:24:32.005815Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T132431904970Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T132431904970Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d50faacb2a0bef198ee21fb926f9c872750b44f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T132431904970Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fa35bc563ee400979ac53c6a803dbd4af87e701f95fbd989245ac78479dcac5 +size 5638892 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T132432915106Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T132432915106Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d8e7b4494162afa02c041b5d624be97dd62742ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T132432915106Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855763968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103307860480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2698379264, + "inactive_split_bytes_peak": 4140574208, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:32.914697Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855763968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103307860480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2692087808, + "inactive_split_bytes_peak": 4140574208, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:32.020369Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2698379264, + "inactive_split_bytes_current_delta": 6291456, + "inactive_split_bytes_peak": 4140574208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 105115549696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105115549696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 609, + "segment_count_current_delta": 0, + "segment_count_peak": 609, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T132432915106Z.pickle", + "timestamp_utc": "2026-08-01T13:24:33.015910Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T132432915106Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T132432915106Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ff7da2db2f107113f1ef8a354966bcec12b11d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T132432915106Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d798258b6e51ccced13275653ff92c1b426a768f982d4f1d7d65c385987d492 +size 5638789 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T132433915200Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T132433915200Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e1947cc613e35762d6b175dbf08d3a2aa9082f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T132433915200Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855763968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103307860480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2694184960, + "inactive_split_bytes_peak": 4140574208, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:33.914732Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855763968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103307860480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2698379264, + "inactive_split_bytes_peak": 4140574208, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:33.030394Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2694184960, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 4140574208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 105115549696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105115549696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 609, + "segment_count_current_delta": 0, + "segment_count_peak": 609, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T132433915200Z.pickle", + "timestamp_utc": "2026-08-01T13:24:34.017073Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T132433915200Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T132433915200Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7dcba508879215a5984bc24e347bb59916541b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T132433915200Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fa5b33374a5e5740771b1c1c27c1679dbdb0d85a5f582d4a7e287c8262fb89d +size 5638790 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T132526823615Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T132526823615Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36bfa1f1dcb1acee8a49fa60310aabb2b0c67f7a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T132526823615Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66805121024, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103307860480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66805121024, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66805121024, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2421866496, + "inactive_split_bytes_peak": 4140574208, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:25:26.823124Z", + "world_size": 8 + }, + "allocated_bytes_current": 66805121024, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103307860480, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855763968, + "active_bytes_peak": 103307860480, + "allocated_bytes_current": 66855763968, + "allocated_bytes_peak": 103307860480, + "device": 4, + "inactive_split_bytes_current": 2694184960, + "inactive_split_bytes_peak": 4140574208, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 105115549696, + "reserved_bytes_peak": 105115549696, + "segment_count_current": 609, + "segment_count_peak": 609, + "timestamp_utc": "2026-08-01T13:24:34.031619Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2421866496, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 4140574208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 105115549696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105115549696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 609, + "segment_count_current_delta": 0, + "segment_count_peak": 609, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T132526823615Z.pickle", + "timestamp_utc": "2026-08-01T13:25:27.032926Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T132526823615Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T132526823615Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d5a02d1fa7b8c38cfcbdc2afc61cd1e5fc34bc3e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T132526823615Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d39f3fc478746e3358b5a3f4f45facb129b62f12da8ee6142f6ee65f35192548 +size 9681161 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T132425383240Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T132425383240Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cae60b68fddfa2bbf170bbb42bfd8f3d3b6f4835 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T132425383240Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 31623011328, + "active_bytes_peak": 79671957504, + "active_bytes_peak_delta": 44205621248, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 79671957504, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 79671957504, + "device": 5, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_peak": 3471774720, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 96016007168, + "reserved_bytes_peak": 96016007168, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T13:24:25.382656Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 31623011328, + "allocated_bytes_peak": 79671957504, + "allocated_bytes_peak_delta": 44205621248, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:23:15.731646Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_current_delta": 2782864384, + "inactive_split_bytes_peak": 3471774720, + "inactive_split_bytes_peak_delta": 3311388672, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 96016007168, + "reserved_bytes_current_delta": 56772001792, + "reserved_bytes_peak": 96016007168, + "reserved_bytes_peak_delta": 56772001792, + "segment_count_current": 550, + "segment_count_current_delta": 479, + "segment_count_peak": 550, + "segment_count_peak_delta": 479, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T132425383240Z.pickle", + "timestamp_utc": "2026-08-01T13:24:25.496530Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T132425383240Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T132425383240Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d33b06f984590bf3f0be39667bac26751e265eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T132425383240Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd21fd5b30bbb4b76c62f24d04febad16329e7205c92d20b953187592e8414c6 +size 5392652 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T132426425071Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T132426425071Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f127f3de713d1f311b32e37347bf09fd218daf8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T132426425071Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 23643517440, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_peak": 3471774720, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:26.424647Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 23643517440, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 79671957504, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 79671957504, + "device": 5, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_peak": 3471774720, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 96016007168, + "reserved_bytes_peak": 96016007168, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T13:24:25.515409Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_current_delta": -224395264, + "inactive_split_bytes_peak": 3471774720, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104870182912, + "reserved_bytes_current_delta": 8854175744, + "reserved_bytes_peak": 104870182912, + "reserved_bytes_peak_delta": 8854175744, + "segment_count_current": 605, + "segment_count_current_delta": 55, + "segment_count_peak": 605, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T132426425071Z.pickle", + "timestamp_utc": "2026-08-01T13:24:26.539359Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T132426425071Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T132426425071Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6a98836443a4ed75317f983a18258896c86db06e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T132426425071Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f9c60d838bca1de4980d243f2ddd5e4afc01c46d8270f43e040c95ea4ab4bb1 +size 5677748 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T132427414233Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T132427414233Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a75f8829c40321916a024da2fec4b5eeabc7313 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T132427414233Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4367634944, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:27.413800Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_peak": 3471774720, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:26.554671Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 895860224, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104870182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104870182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T132427414233Z.pickle", + "timestamp_utc": "2026-08-01T13:24:27.520337Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T132427414233Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T132427414233Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f841981d7844234b8fb070df5ec678a105c373f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T132427414233Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48e63c19fffde7b1915efa3063f8ff4ab11b944a00d78ddd6d03d9719eb9ea1f +size 5632189 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T132428415102Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T132428415102Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b630d0d145fc0e31df496a0cb02f286b4fe6c53d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T132428415102Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2919877632, + "inactive_split_bytes_peak": 4367634944, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:28.414710Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4367634944, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:27.535037Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2919877632, + "inactive_split_bytes_current_delta": 224395264, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104870182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104870182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T132428415102Z.pickle", + "timestamp_utc": "2026-08-01T13:24:28.525606Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T132428415102Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T132428415102Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..549551cd17390f549a6be64c0f1ba3d780d34cb7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T132428415102Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b61d5660b66393ef8c2e78734b5e055afe57f3e4db276a03551434c59ed53382 +size 5631872 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T132429897082Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T132429897082Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac15277b2efb0d492f2e5afc226657da03f1b7d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T132429897082Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4367634944, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:29.896673Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2919877632, + "inactive_split_bytes_peak": 4367634944, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:28.543125Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104870182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104870182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T132429897082Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.005032Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T132429897082Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T132429897082Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..955134ace4f7eccd833556262c83312f0da35862 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T132429897082Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2987b23e283f4c526a4547b0648c3698bcd06b659216bab9d5549b38107705fd +size 5631729 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T132430888032Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T132430888032Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0666ea5cab6fa30eacb5c981439829330adc268c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T132430888032Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_peak": 4367634944, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:30.887635Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4367634944, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:30.019660Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104870182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104870182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T132430888032Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.992594Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T132430888032Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T132430888032Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0998fe73d05d07d7bee6393866f59df09f7320d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T132430888032Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ede2ec6dd4e00da4fcfe5f241bdac67cb60343c109ed21777471e7bce481ae5d +size 5631909 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T132431904985Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T132431904985Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3ed15248697914789d81a6d1c30b289f0337a3e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T132431904985Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2924071936, + "inactive_split_bytes_peak": 4367634944, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:31.904587Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_peak": 4367634944, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:31.007153Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2924071936, + "inactive_split_bytes_current_delta": 224395264, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104870182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104870182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T132431904985Z.pickle", + "timestamp_utc": "2026-08-01T13:24:32.005624Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T132431904985Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T132431904985Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d74cb04086da580c9d1679187e34441595f01a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T132431904985Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c22cf3a9067070e139c4da9bff4a062f9581f80c756927e3a5e01c631b669ac +size 5631751 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T132432915108Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T132432915108Z.json new file mode 100644 index 0000000000000000000000000000000000000000..56a30831073d0e2c41e913cd7eecdb96527b4550 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T132432915108Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2930363392, + "inactive_split_bytes_peak": 4367634944, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:32.914709Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2924071936, + "inactive_split_bytes_peak": 4367634944, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:32.018332Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2930363392, + "inactive_split_bytes_current_delta": 6291456, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104870182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104870182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T132432915108Z.pickle", + "timestamp_utc": "2026-08-01T13:24:33.020297Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T132432915108Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T132432915108Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14da0bb3e0493f9bfc0f0e9092b615603a4cb4cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T132432915108Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48a2e101dc4f0d0f0715a592a8a251351a050cedd18978ed78a3bb0e110bfe1d +size 5631723 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T132433915237Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T132433915237Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd5f8b4728a66065cf5dd32ed044d0382ebe0005 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T132433915237Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_peak": 4367634944, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:33.914755Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2930363392, + "inactive_split_bytes_peak": 4367634944, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:33.034590Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104870182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104870182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T132433915237Z.pickle", + "timestamp_utc": "2026-08-01T13:24:34.016957Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T132433915237Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T132433915237Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0f3262575f2f32993eee23e25515aae9880e6769 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T132433915237Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3adb7bae38e31caf43892ead96d86aee0f261bbe0fcf0619a2eb11daf38c509 +size 5631685 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T132526823585Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T132526823585Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1374cd92dd1b7cb2b859fe3b5bf02d022d220499 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T132526823585Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803823616, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103315474944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803823616, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66803823616, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2418969600, + "inactive_split_bytes_peak": 4367634944, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:25:26.823109Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803823616, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103315474944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103315474944, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103315474944, + "device": 5, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_peak": 4367634944, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 104870182912, + "reserved_bytes_peak": 104870182912, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:34.031231Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2418969600, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 4367634944, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 104870182912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104870182912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T132526823585Z.pickle", + "timestamp_utc": "2026-08-01T13:25:27.031726Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T132526823585Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T132526823585Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6639584f3a21dda439b0ea4015568420549ea504 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T132526823585Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11819fc09b0bc5caaca5f72cfa17c4e87bc6ed09e1f0bcefa08d4e6ac408a7dc +size 10221034 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T132425383239Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T132425383239Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6777cd850a3614e3b3edea6e3e10627c633a31cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T132425383239Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 31623011328, + "active_bytes_peak": 79670515200, + "active_bytes_peak_delta": 44204178944, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 79670515200, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 79670515200, + "device": 6, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_peak": 3884651520, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 95546245120, + "reserved_bytes_peak": 95546245120, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T13:24:25.382643Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 31623011328, + "allocated_bytes_peak": 79670515200, + "allocated_bytes_peak_delta": 44204178944, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:23:15.731602Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_current_delta": 2782864384, + "inactive_split_bytes_peak": 3884651520, + "inactive_split_bytes_peak_delta": 3724265472, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 95546245120, + "reserved_bytes_current_delta": 56302239744, + "reserved_bytes_peak": 95546245120, + "reserved_bytes_peak_delta": 56302239744, + "segment_count_current": 543, + "segment_count_current_delta": 472, + "segment_count_peak": 543, + "segment_count_peak_delta": 472, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T132425383239Z.pickle", + "timestamp_utc": "2026-08-01T13:24:25.494445Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T132425383239Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T132425383239Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e2d5a64a00391861d2b2d500b12feb861ebde71f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T132425383239Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4ceec2471c23c571174b8b5395124ff3cfbea2dd35d4f5159a90628009496d0 +size 5391526 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T132426425050Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T132426425050Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9e667d057da4bdd0b618397c04efc04532b8ea55 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T132426425050Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 23643919872, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_peak": 3884651520, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:26.424639Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 23643919872, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 79670515200, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 79670515200, + "device": 6, + "inactive_split_bytes_current": 2915683328, + "inactive_split_bytes_peak": 3884651520, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 95546245120, + "reserved_bytes_peak": 95546245120, + "segment_count_current": 543, + "segment_count_peak": 543, + "timestamp_utc": "2026-08-01T13:24:25.513689Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_current_delta": -224395264, + "inactive_split_bytes_peak": 3884651520, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 9558818816, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 9558818816, + "segment_count_current": 605, + "segment_count_current_delta": 62, + "segment_count_peak": 605, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T132426425050Z.pickle", + "timestamp_utc": "2026-08-01T13:24:26.541217Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T132426425050Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T132426425050Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f28cbb041487e40a77d679331852af198bce65c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T132426425050Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:048c636ee840fef3edb770dbef9ccc438a483da3b564fdc094e9c0ff4e916eb9 +size 5677149 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T132427414231Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T132427414231Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a52ea34e03a4fc1761444f00c31f841a5739c96e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T132427414231Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4075589632, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:27.413819Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2691288064, + "inactive_split_bytes_peak": 3884651520, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:26.556637Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 4075589632, + "inactive_split_bytes_peak_delta": 190938112, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T132427414231Z.pickle", + "timestamp_utc": "2026-08-01T13:24:27.524871Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T132427414231Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T132427414231Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5294aba4e479fd1bf397791c8aa0588a23177649 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T132427414231Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8d0d9c4ab3b8da4d829fd1f60bda47aa2eb18fb523959b265a19486e93bc4a2 +size 5632505 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T132428415102Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T132428415102Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0109b135f89e6469cffc80ab830c6b0c7b822c50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T132428415102Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2684996608, + "inactive_split_bytes_peak": 4075589632, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:28.414693Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4075589632, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:27.542291Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2684996608, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 4075589632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T132428415102Z.pickle", + "timestamp_utc": "2026-08-01T13:24:28.517253Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T132428415102Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T132428415102Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9785ded8a1897b03d46f17b0eff7ba05f3a95563 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T132428415102Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9f14eb29aab4e19464f287596ff80efa6834f732ea2582b1c773127c4b403a2 +size 5632150 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T132429897073Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T132429897073Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bd0def83693127d0d0080f6b97376f73d3b753e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T132429897073Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4075589632, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:29.896663Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2684996608, + "inactive_split_bytes_peak": 4075589632, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:28.532114Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 4075589632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T132429897073Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.001354Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T132429897073Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T132429897073Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..50bd2bbf1fd5f51cbcb2eaf48ba4ef7617e08122 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T132429897073Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35d90baf65da99cce75c9a2da9b0e6b7878f2907742acda732eb281d943ced9d +size 5632115 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T132430888030Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T132430888030Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a02612d27274b44c3911481d670e60c1c75431c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T132430888030Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_peak": 4075589632, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:30.887623Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2689190912, + "inactive_split_bytes_peak": 4075589632, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:30.016026Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 4075589632, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T132430888030Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.989000Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T132430888030Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T132430888030Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b287427de9b8bbdbbb592e4790eaf59c0f343080 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T132430888030Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c58d0dd74ce61ae2c286f6502dfd95e0175c8990efcacaa4163fa0c878a3cd6 +size 5632371 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T132431904999Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T132431904999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..502e57f9f0b40fbc5640a22df0b35a4da3b097b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T132431904999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2924071936, + "inactive_split_bytes_peak": 4169252864, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:31.904588Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2699676672, + "inactive_split_bytes_peak": 4075589632, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:31.003423Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2924071936, + "inactive_split_bytes_current_delta": 224395264, + "inactive_split_bytes_peak": 4169252864, + "inactive_split_bytes_peak_delta": 93663232, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T132431904999Z.pickle", + "timestamp_utc": "2026-08-01T13:24:32.005960Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T132431904999Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T132431904999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d42a8095d3bdfc5dcaad6f427a6cd3bb82b121fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T132431904999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:279a0b9bfe7658cde76d8744ddafd86eaf1e472d6552716c8d3a857300015680 +size 5632213 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T132432915145Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T132432915145Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2806df5ba9ffa978df8dcc85862048b30b842a01 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T132432915145Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4169252864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:32.914726Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2924071936, + "inactive_split_bytes_peak": 4169252864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:32.020344Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_current_delta": -228589568, + "inactive_split_bytes_peak": 4169252864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T132432915145Z.pickle", + "timestamp_utc": "2026-08-01T13:24:33.014495Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T132432915145Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T132432915145Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a3483733f24d4d57af611b5bf8039b6be016c005 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T132432915145Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a276424ce4391c95e256845347dbcd026c6bc89b0bcc244a054c7e681d9d09c +size 5632132 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T132433915193Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T132433915193Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab3d546da4a8b9e3d45f0e8c9c622436642a40f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T132433915193Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2926169088, + "inactive_split_bytes_peak": 4171350016, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:33.914714Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2695482368, + "inactive_split_bytes_peak": 4169252864, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:33.028883Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2926169088, + "inactive_split_bytes_current_delta": 230686720, + "inactive_split_bytes_peak": 4171350016, + "inactive_split_bytes_peak_delta": 2097152, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T132433915193Z.pickle", + "timestamp_utc": "2026-08-01T13:24:34.013979Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T132433915193Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T132433915193Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e09c9de03b0072fe6c6d3a17caf7ff947ff89830 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T132433915193Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c07650845927965c6d0c4f724949d4847326cacbb03ddb5a8632b33043546d84 +size 5632184 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T132526823650Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T132526823650Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d79f08bd4a787b53ac9ee93880ec3119eb18175 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T132526823650Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803823616, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103314435072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803823616, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66803823616, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2418969600, + "inactive_split_bytes_peak": 4171350016, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:25:26.823103Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803823616, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103314435072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 103314435072, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 103314435072, + "device": 6, + "inactive_split_bytes_current": 2926169088, + "inactive_split_bytes_peak": 4171350016, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 105105063936, + "reserved_bytes_peak": 105105063936, + "segment_count_current": 605, + "segment_count_peak": 605, + "timestamp_utc": "2026-08-01T13:24:34.028219Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2418969600, + "inactive_split_bytes_current_delta": -507199488, + "inactive_split_bytes_peak": 4171350016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 105105063936, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 105105063936, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 605, + "segment_count_current_delta": 0, + "segment_count_peak": 605, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T132526823650Z.pickle", + "timestamp_utc": "2026-08-01T13:25:27.018423Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T132526823650Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T132526823650Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7460c1a7d81334b5752465e3702a4fe07ba82175 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T132526823650Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:101907edb5f8dc30cdeed71ae68cb483e2eb24204120718bd4ccd94e3c2b4fdd +size 9674567 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T132425383252Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T132425383252Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96e6c3d2fbba2d860a8c94c5bc7e582c2e4ffc5c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T132425383252Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 31623011840, + "active_bytes_peak": 79670515200, + "active_bytes_peak_delta": 44204178944, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 79670515200, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 79670515200, + "device": 7, + "inactive_split_bytes_current": 2915682816, + "inactive_split_bytes_peak": 3770086400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 95076483072, + "reserved_bytes_peak": 95076483072, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T13:24:25.382781Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 31623011840, + "allocated_bytes_peak": 79670515200, + "allocated_bytes_peak_delta": 44204178944, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:23:15.731601Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2915682816, + "inactive_split_bytes_current_delta": 2782863872, + "inactive_split_bytes_peak": 3770086400, + "inactive_split_bytes_peak_delta": 3609700352, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 95076483072, + "reserved_bytes_current_delta": 55832477696, + "reserved_bytes_peak": 95076483072, + "reserved_bytes_peak_delta": 55832477696, + "segment_count_current": 541, + "segment_count_current_delta": 470, + "segment_count_peak": 541, + "segment_count_peak_delta": 470, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T132425383252Z.pickle", + "timestamp_utc": "2026-08-01T13:24:25.495594Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T132425383252Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T132425383252Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..00239bac6a286428f877825e275529d6a9a546a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T132425383252Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a1850a0397bff76f3da78e2beb4ccba013cf8c675c9931d7c9af122f9d6d091 +size 5394296 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T132426425045Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T132426425045Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0110e99a466cc33d4e0aa6be6a2a1c40021a1c99 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T132426425045Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314476544, + "active_bytes_peak_delta": 23643961344, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2691287552, + "inactive_split_bytes_peak": 3770086400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:26.424636Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314476544, + "allocated_bytes_peak_delta": 23643961344, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 79670515200, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 79670515200, + "device": 7, + "inactive_split_bytes_current": 2915682816, + "inactive_split_bytes_peak": 3770086400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 95076483072, + "reserved_bytes_peak": 95076483072, + "segment_count_current": 541, + "segment_count_peak": 541, + "timestamp_utc": "2026-08-01T13:24:25.514346Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2691287552, + "inactive_split_bytes_current_delta": -224395264, + "inactive_split_bytes_peak": 3770086400, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 9558818816, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 9558818816, + "segment_count_current": 603, + "segment_count_current_delta": 62, + "segment_count_peak": 603, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T132426425045Z.pickle", + "timestamp_utc": "2026-08-01T13:24:26.537752Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T132426425045Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T132426425045Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e478f50dacd744974252426b5396ab746cc4811c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T132426425045Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:309c2b2519fa5b088529791914bcab34ce8802d4143ce15413b08483aa172483 +size 5678553 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T132427414220Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T132427414220Z.json new file mode 100644 index 0000000000000000000000000000000000000000..392398aef1c18734050e923623f21ffa731b3654 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T132427414220Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314476544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2695481856, + "inactive_split_bytes_peak": 4075589120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:27.413810Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314476544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2691287552, + "inactive_split_bytes_peak": 3770086400, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:26.555050Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2695481856, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 4075589120, + "inactive_split_bytes_peak_delta": 305502720, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T132427414220Z.pickle", + "timestamp_utc": "2026-08-01T13:24:27.518771Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T132427414220Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T132427414220Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ded82da0658c2f1e4416522fa12ef09b99a8e50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T132427414220Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99c069b5a1687fb08a812daad5124baee39c54240fce049c8ec8db1a30494c12 +size 5633838 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T132428415228Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T132428415228Z.json new file mode 100644 index 0000000000000000000000000000000000000000..13fb641fcc67c78f9cead28cd8ab6f982815e85d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T132428415228Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314476544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2919877120, + "inactive_split_bytes_peak": 4150202368, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:28.414788Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314476544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2695481856, + "inactive_split_bytes_peak": 4075589120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:27.533657Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2919877120, + "inactive_split_bytes_current_delta": 224395264, + "inactive_split_bytes_peak": 4150202368, + "inactive_split_bytes_peak_delta": 74613248, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T132428415228Z.pickle", + "timestamp_utc": "2026-08-01T13:24:28.519211Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T132428415228Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T132428415228Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d2aec31585703dc80840ccf6c6bf0f754a70552b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T132428415228Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc2e43199368a4421ad4c8d40e7b1a3754b2f7f3a3ff0a2b72312a9033aaec3a +size 5633377 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T132429897183Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T132429897183Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ec5ed2fc6f71e4df916f61b470895dfb0ee820b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T132429897183Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314476544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2689190400, + "inactive_split_bytes_peak": 4150202368, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:29.896735Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314476544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2919877120, + "inactive_split_bytes_peak": 4150202368, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:28.534157Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2689190400, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 4150202368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T132429897183Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.004255Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T132429897183Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T132429897183Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf812663b2fba1ab56066b5b457d1c82b083e9e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T132429897183Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acdc4e3282b5c2d5ac5c779bac3335b080c261fc5485674fdb3cd6bcaf69b210 +size 5633309 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T132430888174Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T132430888174Z.json new file mode 100644 index 0000000000000000000000000000000000000000..646311b77b348bc0be9b8f83dc79765988b04f7a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T132430888174Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314476544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2699676160, + "inactive_split_bytes_peak": 4150202368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:30.887685Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314476544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2689190400, + "inactive_split_bytes_peak": 4150202368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:30.019157Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2699676160, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 4150202368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T132430888174Z.pickle", + "timestamp_utc": "2026-08-01T13:24:30.992238Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T132430888174Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T132430888174Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1649f3a434756d2d44ce9a0785b7e0fce2b3ae14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T132430888174Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5be14ecf47f271c9662442e63d5fee684fa9c6947b5e6e1a887bfc0398fe838 +size 5633524 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T132431905056Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T132431905056Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4bbd267a0807b03cedef630294d2eaed32ed5f80 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T132431905056Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314476544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2689190400, + "inactive_split_bytes_peak": 4150202368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:31.904653Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314476544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2699676160, + "inactive_split_bytes_peak": 4150202368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:31.006875Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2689190400, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 4150202368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T132431905056Z.pickle", + "timestamp_utc": "2026-08-01T13:24:32.007525Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T132431905056Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T132431905056Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc79dbd3897ab97be1f74f698966ecd3f3333c56 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T132431905056Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8252f50331a9f4c013dc50c089fabaa54fb7f53802626fd008d2d96cacd78ccb +size 5633412 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T132432915167Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T132432915167Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f62689cb3ccb0faa0a5c431b080e7e440ead4bc0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T132432915167Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314476544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2695481856, + "inactive_split_bytes_peak": 4150202368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:32.914756Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314476544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2689190400, + "inactive_split_bytes_peak": 4150202368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:32.022380Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2695481856, + "inactive_split_bytes_current_delta": 6291456, + "inactive_split_bytes_peak": 4150202368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T132432915167Z.pickle", + "timestamp_utc": "2026-08-01T13:24:33.018679Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T132432915167Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T132432915167Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b687d28a4078629102fc75107f68ec890c3d7ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T132432915167Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ac2e55fac2d9ca5f7287d4b31da274052d1d4a510cf7363ad6a8d1893c56b4a +size 5633383 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T132433915191Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T132433915191Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15609834cf7e2ecae1e66a43826a005827c8317b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T132433915191Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103314476544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2691287552, + "inactive_split_bytes_peak": 4150202368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:33.914790Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103314476544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2695481856, + "inactive_split_bytes_peak": 4150202368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:33.033458Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2691287552, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 4150202368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T132433915191Z.pickle", + "timestamp_utc": "2026-08-01T13:24:34.019335Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T132433915191Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T132433915191Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7d70e5316e316e5ea405fc3ccc32d091e769bca1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T132433915191Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86854956733cb6a76cb0858fe2301776505237d3f1dfbd70324474e7da7e6abd +size 5633350 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T132526823585Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T132526823585Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c3012eda9c306ade67c53707268744e447e1452f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T132526823585Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803824128, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 103314476544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803824128, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66803824128, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2418969088, + "inactive_split_bytes_peak": 4150202368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:25:26.823110Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803824128, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 103314476544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 103314476544, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 103314476544, + "device": 7, + "inactive_split_bytes_current": 2691287552, + "inactive_split_bytes_peak": 4150202368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 104635301888, + "reserved_bytes_peak": 104635301888, + "segment_count_current": 603, + "segment_count_peak": 603, + "timestamp_utc": "2026-08-01T13:24:34.034249Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2418969088, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 4150202368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 104635301888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 104635301888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 603, + "segment_count_current_delta": 0, + "segment_count_peak": 603, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T132526823585Z.pickle", + "timestamp_utc": "2026-08-01T13:25:27.033985Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T132526823585Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T132526823585Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..54c605a07c4a4f7bbced50a0181e0a3d3e5d3fed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T132526823585Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37e730bcb77d146f12dc482403e325c4ef60df3e854390270803779a56990980 +size 9675771 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..5f791f8c95e1553e8790e3c6dae9e13b744a2b5b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 2048 +capacity: 1.00 +expert_dispatch_overlap: on +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6027 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 225 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T132816253387Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T132816253387Z.json new file mode 100644 index 0000000000000000000000000000000000000000..40813e90e87bb411ffedd840496f3cdcfa801aed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T132816253387Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442647552, + "active_bytes_current_delta": 37211192320, + "active_bytes_peak": 79923207680, + "active_bytes_peak_delta": 44456871424, + "after": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 79923207680, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 79570886144, + "device": 0, + "inactive_split_bytes_current": 2878663680, + "inactive_split_bytes_peak": 5559340544, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 96338968576, + "reserved_bytes_peak": 96338968576, + "segment_count_current": 508, + "segment_count_peak": 508, + "timestamp_utc": "2026-08-01T13:28:16.252740Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442647552, + "allocated_bytes_current_delta": 37211192320, + "allocated_bytes_peak": 79570886144, + "allocated_bytes_peak_delta": 44104549888, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:27:07.177708Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2878663680, + "inactive_split_bytes_current_delta": 2743747584, + "inactive_split_bytes_peak": 5559340544, + "inactive_split_bytes_peak_delta": 5398708736, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 96338968576, + "reserved_bytes_current_delta": 57092866048, + "reserved_bytes_peak": 96338968576, + "reserved_bytes_peak_delta": 57092866048, + "segment_count_current": 508, + "segment_count_current_delta": 436, + "segment_count_peak": 508, + "segment_count_peak_delta": 436, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T132816253387Z.pickle", + "timestamp_utc": "2026-08-01T13:28:16.371386Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T132816253387Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T132816253387Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0d68533b67d99926415bf40d09c490d58ba1beb9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T132816253387Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f422130ecaab6cb0b352ddbe61ccf82503538150580bb30256f5a03cd8c097d +size 5956872 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T132817404410Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T132817404410Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f5625c204a7aed166451b1661bfc061da199015 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T132817404410Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442647552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103557437440, + "active_bytes_peak_delta": 23634229760, + "after": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 2817846272, + "inactive_split_bytes_peak": 5559340544, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:17.403981Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442647552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205115904, + "allocated_bytes_peak_delta": 23634229760, + "before": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 79923207680, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 79570886144, + "device": 0, + "inactive_split_bytes_current": 2878663680, + "inactive_split_bytes_peak": 5559340544, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 96338968576, + "reserved_bytes_peak": 96338968576, + "segment_count_current": 508, + "segment_count_peak": 508, + "timestamp_utc": "2026-08-01T13:28:16.388364Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 2817846272, + "inactive_split_bytes_current_delta": -60817408, + "inactive_split_bytes_peak": 5559340544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_current_delta": 10034872320, + "reserved_bytes_peak": 106373840896, + "reserved_bytes_peak_delta": 10034872320, + "segment_count_current": 579, + "segment_count_current_delta": 71, + "segment_count_peak": 579, + "segment_count_peak_delta": 71, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T132817404410Z.pickle", + "timestamp_utc": "2026-08-01T13:28:17.520716Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T132817404410Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T132817404410Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..25a0a52bb67bbaca4a67e1a0eebc18d7597d682f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T132817404410Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b0e26ac21f1df124a507e57647d5d33b2c248a9fdd53cc19437bcfbde554f16 +size 6662639 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T132818527000Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T132818527000Z.json new file mode 100644 index 0000000000000000000000000000000000000000..498513dde99f4116d319a78ef6e201c4571d2975 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T132818527000Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442647552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103557437440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 4090817536, + "inactive_split_bytes_peak": 5559340544, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:18.526581Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442647552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205115904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 2817846272, + "inactive_split_bytes_peak": 5559340544, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:17.535178Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4090817536, + "inactive_split_bytes_current_delta": 1272971264, + "inactive_split_bytes_peak": 5559340544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106373840896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 579, + "segment_count_current_delta": 0, + "segment_count_peak": 579, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T132818527000Z.pickle", + "timestamp_utc": "2026-08-01T13:28:18.635102Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T132818527000Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T132818527000Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..09750b1e6a79f59d5627d95a04eacfad32cd7e60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T132818527000Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e4bc673e462cbc4c90b1e3991721313dbad46a1a09f32322194a62e6e904ce7 +size 6662562 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T132819923724Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T132819923724Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2da49961f88971053106bcd604aa49f1912f8be2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T132819923724Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442647552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103557437440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 3855936512, + "inactive_split_bytes_peak": 5559340544, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:19.923328Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442647552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205115904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 4090817536, + "inactive_split_bytes_peak": 5559340544, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:18.649039Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3855936512, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 5559340544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106373840896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 579, + "segment_count_current_delta": 0, + "segment_count_peak": 579, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T132819923724Z.pickle", + "timestamp_utc": "2026-08-01T13:28:20.035784Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T132819923724Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T132819923724Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..16e9894793fb316ae8f2615e68a2ae7333e8b455 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T132819923724Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c085ce0b5fa0ceaeb4b15c8951c86b0361b4e09efdc658ac44c3a2b6d3cbd1ce +size 6662592 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T132821308045Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T132821308045Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c9ce9ecb01c934e8f21bfc29e98c7418735f6705 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T132821308045Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442647552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103557437440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 3855936512, + "inactive_split_bytes_peak": 5559340544, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:21.307617Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442647552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205115904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 3855936512, + "inactive_split_bytes_peak": 5559340544, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:20.050285Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3855936512, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5559340544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106373840896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 579, + "segment_count_current_delta": 0, + "segment_count_peak": 579, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T132821308045Z.pickle", + "timestamp_utc": "2026-08-01T13:28:21.423089Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T132821308045Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T132821308045Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bcf128ef8ddda12c86e284d0015e14599722bfa4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T132821308045Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc8fd7037111ea1f02695788a76818ff45c164387cfb97a6b041e0e72aab5c6e +size 6662560 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T132822447977Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T132822447977Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c137433069dead19a884cfcb8261fe3891c61685 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T132822447977Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442647552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103557437440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 3386174464, + "inactive_split_bytes_peak": 5559340544, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:22.447561Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442647552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205115904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 3855936512, + "inactive_split_bytes_peak": 5559340544, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:21.443795Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3386174464, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 5559340544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106373840896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 579, + "segment_count_current_delta": 0, + "segment_count_peak": 579, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T132822447977Z.pickle", + "timestamp_utc": "2026-08-01T13:28:22.555686Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T132822447977Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T132822447977Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c8ceb0949de5b614385298cf5b12b0b00edcc37f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T132822447977Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f156abac0c58648881be0061577c86d43967ac4be28a13ce1ca33395b949b5d +size 6662521 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T132823575790Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T132823575790Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8bcd2797acde53fb56fed4cfa1016354a72c744a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T132823575790Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442647552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103557437440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 3855936512, + "inactive_split_bytes_peak": 5559340544, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:23.575376Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442647552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205115904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 3386174464, + "inactive_split_bytes_peak": 5559340544, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:22.569245Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3855936512, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 5559340544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106373840896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 579, + "segment_count_current_delta": 0, + "segment_count_peak": 579, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T132823575790Z.pickle", + "timestamp_utc": "2026-08-01T13:28:23.681036Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T132823575790Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T132823575790Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b22d887fe015251cbf1bb5db932d8fdc91dcd440 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T132823575790Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfcdccb2d9887739a4e7d8dd668e8977d3b616dc68ac34bb2c2656b8ffd2b3d9 +size 6662533 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T132824879771Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T132824879771Z.json new file mode 100644 index 0000000000000000000000000000000000000000..addee742fb20012f3ef7c781a8bb4f1b24506c6a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T132824879771Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442647552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103557437440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 4090817536, + "inactive_split_bytes_peak": 5559340544, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:24.879358Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442647552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205115904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 3855936512, + "inactive_split_bytes_peak": 5559340544, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:23.694691Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4090817536, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 5559340544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106373840896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 579, + "segment_count_current_delta": 0, + "segment_count_peak": 579, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T132824879771Z.pickle", + "timestamp_utc": "2026-08-01T13:28:24.986331Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T132824879771Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T132824879771Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3589815c25a7f53b766cd585821226e883020e92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T132824879771Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:878b686ca8a6ba500fdec7eb5cee1e58ea777f562af8bf555252ed8eeee4139d +size 6662573 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T132826001225Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T132826001225Z.json new file mode 100644 index 0000000000000000000000000000000000000000..37dd69f8f5ad8883c9b0b1f976ff733ad2b40419 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T132826001225Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442647552, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103557437440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 4090817536, + "inactive_split_bytes_peak": 5559340544, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:26.000801Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442647552, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103205115904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 4090817536, + "inactive_split_bytes_peak": 5559340544, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:24.999860Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4090817536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5559340544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106373840896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 579, + "segment_count_current_delta": 0, + "segment_count_peak": 579, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T132826001225Z.pickle", + "timestamp_utc": "2026-08-01T13:28:26.106957Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T132826001225Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T132826001225Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0c086703242c2d7fded324e4f552be256044e420 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T132826001225Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb77c8c00908d73d95ac8b3d6d4ecec0555ba3ca4381111b9aaeb4acec42ef15 +size 6662542 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T132921090501Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T132921090501Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c608ce32002e35c5188caf43266c9c6ee15bc8f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T132921090501Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61168358400, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103557437440, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61168358400, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 61168358400, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 3757370368, + "inactive_split_bytes_peak": 5559340544, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:29:21.090023Z", + "world_size": 8 + }, + "allocated_bytes_current": 61168358400, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103205115904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442647552, + "active_bytes_peak": 103557437440, + "allocated_bytes_current": 72442647552, + "allocated_bytes_peak": 103205115904, + "device": 0, + "inactive_split_bytes_current": 4090817536, + "inactive_split_bytes_peak": 5559340544, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_peak": 106373840896, + "segment_count_current": 579, + "segment_count_peak": 579, + "timestamp_utc": "2026-08-01T13:28:26.120958Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3757370368, + "inactive_split_bytes_current_delta": -333447168, + "inactive_split_bytes_peak": 5559340544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106373840896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106373840896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 579, + "segment_count_current_delta": 0, + "segment_count_peak": 579, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T132921090501Z.pickle", + "timestamp_utc": "2026-08-01T13:29:21.289250Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T132921090501Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T132921090501Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a0b2f2acaa06366ae8188093aeb939c996158c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T132921090501Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:179360d0e5c4cd683456b3616e33511d66e4d06902ae2f28ccc9370677d758a9 +size 10559208 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T132816253412Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T132816253412Z.json new file mode 100644 index 0000000000000000000000000000000000000000..df458c21e1eaf1bbfa5bda598a107b6a907deb5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T132816253412Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 37209512960, + "active_bytes_peak": 80967250944, + "active_bytes_peak_delta": 45500914688, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 80967250944, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 79569307648, + "device": 1, + "inactive_split_bytes_current": 2920188928, + "inactive_split_bytes_peak": 5674634752, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 96571752448, + "reserved_bytes_peak": 96571752448, + "segment_count_current": 507, + "segment_count_peak": 507, + "timestamp_utc": "2026-08-01T13:28:16.252799Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 37209512960, + "allocated_bytes_peak": 79569307648, + "allocated_bytes_peak_delta": 44102971392, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:27:07.177934Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2920188928, + "inactive_split_bytes_current_delta": 2787369984, + "inactive_split_bytes_peak": 5674634752, + "inactive_split_bytes_peak_delta": 5514248704, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 96571752448, + "reserved_bytes_current_delta": 57327747072, + "reserved_bytes_peak": 96571752448, + "reserved_bytes_peak_delta": 57327747072, + "segment_count_current": 507, + "segment_count_current_delta": 436, + "segment_count_peak": 507, + "segment_count_peak_delta": 436, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T132816253412Z.pickle", + "timestamp_utc": "2026-08-01T13:28:16.373715Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T132816253412Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T132816253412Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dae09f5f7e77dc43901c43e075c5f8e68f294312 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T132816253412Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b590684a5eb0f2a9ce34716bdb6dde24978835b4fbf779b2a1eb8f56b5b716a9 +size 5948522 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T132817404390Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T132817404390Z.json new file mode 100644 index 0000000000000000000000000000000000000000..98470e075598cd50dd297fed131ec27e9bff88cf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T132817404390Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103568595456, + "active_bytes_peak_delta": 22601344512, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3450768384, + "inactive_split_bytes_peak": 5674634752, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:17.403981Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103216273920, + "allocated_bytes_peak_delta": 23646966272, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 80967250944, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 79569307648, + "device": 1, + "inactive_split_bytes_current": 2920188928, + "inactive_split_bytes_peak": 5674634752, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 96571752448, + "reserved_bytes_peak": 96571752448, + "segment_count_current": 507, + "segment_count_peak": 507, + "timestamp_utc": "2026-08-01T13:28:16.392429Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3450768384, + "inactive_split_bytes_current_delta": 530579456, + "inactive_split_bytes_peak": 5674634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 9791602688, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 9791602688, + "segment_count_current": 576, + "segment_count_current_delta": 69, + "segment_count_peak": 576, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T132817404390Z.pickle", + "timestamp_utc": "2026-08-01T13:28:17.520584Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T132817404390Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T132817404390Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7c77864ebd88e63f721a625e3edb901fd141c02c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T132817404390Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:008d32c69c77aef7c9555822596f13e5cce656769cf8ecc64ce7ca9a892f36be +size 5975404 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T132818526952Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T132818526952Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03f2ed85cc8e8b43cbac796a42e497ae55641130 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T132818526952Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103568595456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3333327872, + "inactive_split_bytes_peak": 5674634752, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:18.526567Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103216273920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3450768384, + "inactive_split_bytes_peak": 5674634752, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:17.534937Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3333327872, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 5674634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 576, + "segment_count_current_delta": 0, + "segment_count_peak": 576, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T132818526952Z.pickle", + "timestamp_utc": "2026-08-01T13:28:18.638216Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T132818526952Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T132818526952Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2b2e7e481b58e222b290d254623f66f4faf9da7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T132818526952Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c035d258c7671eccb983b40bf04dec27218c0c77bd1e5e1c5e474f664a5b3d31 +size 5972245 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T132819923755Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T132819923755Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8a1b1c3cf2af4ee454e85be2db7070e4128a4c89 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T132819923755Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103568595456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3333327872, + "inactive_split_bytes_peak": 5674634752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:19.923347Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103216273920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3333327872, + "inactive_split_bytes_peak": 5674634752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:18.652278Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3333327872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5674634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 576, + "segment_count_current_delta": 0, + "segment_count_peak": 576, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T132819923755Z.pickle", + "timestamp_utc": "2026-08-01T13:28:20.033439Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T132819923755Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T132819923755Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c90989907297c3085d1011af7262fc21eac49be3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T132819923755Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:380d2d011449877d7f41f441643a54f5cecee7b73c992b25f2666039042a0223 +size 5972267 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T132821307905Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T132821307905Z.json new file mode 100644 index 0000000000000000000000000000000000000000..26009a880a8c00061c8d98adbc81cd1a54ead163 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T132821307905Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103568595456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3098446848, + "inactive_split_bytes_peak": 5674634752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:21.307505Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103216273920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3333327872, + "inactive_split_bytes_peak": 5674634752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:20.047917Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3098446848, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 5674634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 576, + "segment_count_current_delta": 0, + "segment_count_peak": 576, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T132821307905Z.pickle", + "timestamp_utc": "2026-08-01T13:28:21.421957Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T132821307905Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T132821307905Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5bd63f4f23697fdbe7091787715a85b5d50c010c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T132821307905Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cf0329d2a8518583c8690533a3bb492c19aac3258213367ad7b80b32adaf3f9 +size 5972036 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T132822447948Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T132822447948Z.json new file mode 100644 index 0000000000000000000000000000000000000000..30334d6455a1b081c1c9602c7f892a63c321415a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T132822447948Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103568595456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3333327872, + "inactive_split_bytes_peak": 5674634752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:22.447555Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103216273920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3098446848, + "inactive_split_bytes_peak": 5674634752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:21.438168Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3333327872, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 5674634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 576, + "segment_count_current_delta": 0, + "segment_count_peak": 576, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T132822447948Z.pickle", + "timestamp_utc": "2026-08-01T13:28:22.552221Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T132822447948Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T132822447948Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f2573aca74c6647a4c68232342e733216c12edc4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T132822447948Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c2c005440299ffeb379a8b339ba8d800ed6c58dd7c1b07e99f6ef11e206bdbf +size 5972067 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T132823575779Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T132823575779Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03bc16a103b477259c27cfacf07f987a11107b8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T132823575779Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103568595456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3333327872, + "inactive_split_bytes_peak": 5674634752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:23.575382Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103216273920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3333327872, + "inactive_split_bytes_peak": 5674634752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:22.565877Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3333327872, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5674634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 576, + "segment_count_current_delta": 0, + "segment_count_peak": 576, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T132823575779Z.pickle", + "timestamp_utc": "2026-08-01T13:28:23.680275Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T132823575779Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T132823575779Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3cf270e19f1cbad5a9f97fe306569c657ef65df5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T132823575779Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:266f3abc484679828a01ade90b79938a4b3d1ee9e382303bee4fb3ddcdf8db6c +size 5972096 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T132824879759Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T132824879759Z.json new file mode 100644 index 0000000000000000000000000000000000000000..37d4e299a38b3f61e74d998ef11d9c08dfb0fee5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T132824879759Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103568595456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 2802748416, + "inactive_split_bytes_peak": 5674634752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:24.879364Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103216273920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3333327872, + "inactive_split_bytes_peak": 5674634752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:23.693707Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 2802748416, + "inactive_split_bytes_current_delta": -530579456, + "inactive_split_bytes_peak": 5674634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 576, + "segment_count_current_delta": 0, + "segment_count_peak": 576, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T132824879759Z.pickle", + "timestamp_utc": "2026-08-01T13:28:24.983253Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T132824879759Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T132824879759Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b19c06eda2334153b3e333a6da277add95866e0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T132824879759Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:680d95cf1b1178a1eee50add97891d22590358906e7e3db5453671a386e88a63 +size 5971462 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T132826001201Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T132826001201Z.json new file mode 100644 index 0000000000000000000000000000000000000000..914723a9f59bbe6880570b4f83d72b4cd659a3d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T132826001201Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103568595456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3098446848, + "inactive_split_bytes_peak": 5674634752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:26.000802Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103216273920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 2802748416, + "inactive_split_bytes_peak": 5674634752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:24.996532Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3098446848, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 5674634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 576, + "segment_count_current_delta": 0, + "segment_count_peak": 576, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T132826001201Z.pickle", + "timestamp_utc": "2026-08-01T13:28:26.103448Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T132826001201Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T132826001201Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..73a038902a0301fbc2f62ffe808b51843c028722 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T132826001201Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38f4fc86352df356da84e21a54eb8249e6e6a1b963ebb0411e37004c1dff405e +size 5971439 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T132921090514Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T132921090514Z.json new file mode 100644 index 0000000000000000000000000000000000000000..57d0f11195dbbad2bf49038111c14b63b8be3fd6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T132921090514Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166679040, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103568595456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166679040, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 61166679040, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3681455104, + "inactive_split_bytes_peak": 5674634752, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:29:21.090027Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166679040, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103216273920, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103568595456, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103216273920, + "device": 1, + "inactive_split_bytes_current": 3098446848, + "inactive_split_bytes_peak": 5674634752, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 576, + "segment_count_peak": 576, + "timestamp_utc": "2026-08-01T13:28:26.116808Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3681455104, + "inactive_split_bytes_current_delta": 583008256, + "inactive_split_bytes_peak": 5674634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 576, + "segment_count_current_delta": 0, + "segment_count_peak": 576, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T132921090514Z.pickle", + "timestamp_utc": "2026-08-01T13:29:21.289569Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T132921090514Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T132921090514Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bec6556b21827760b699c296d2e6827e65880236 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T132921090514Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:128b12f48bc080bd8686c99995b41e0e8abeba6918962bfbc61e9f2057755f6f +size 9736145 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T132816253850Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T132816253850Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c42d4f6c5aef5f1aef76b2a3ded57ae18d913134 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T132816253850Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 37209512960, + "active_bytes_peak": 79921629184, + "active_bytes_peak_delta": 44455292928, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 79921629184, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 79569307648, + "device": 2, + "inactive_split_bytes_current": 2685307904, + "inactive_split_bytes_peak": 5439753728, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 96101990400, + "reserved_bytes_peak": 96101990400, + "segment_count_current": 506, + "segment_count_peak": 506, + "timestamp_utc": "2026-08-01T13:28:16.253103Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 37209512960, + "allocated_bytes_peak": 79569307648, + "allocated_bytes_peak_delta": 44102971392, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:27:07.178010Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 2685307904, + "inactive_split_bytes_current_delta": 2552488960, + "inactive_split_bytes_peak": 5439753728, + "inactive_split_bytes_peak_delta": 5279367680, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 96101990400, + "reserved_bytes_current_delta": 56857985024, + "reserved_bytes_peak": 96101990400, + "reserved_bytes_peak_delta": 56857985024, + "segment_count_current": 506, + "segment_count_current_delta": 435, + "segment_count_peak": 506, + "segment_count_peak_delta": 435, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T132816253850Z.pickle", + "timestamp_utc": "2026-08-01T13:28:16.390752Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T132816253850Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T132816253850Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4bd44b785a23b95ced7dd6e745eb7c65508a1ee6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T132816253850Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b14d1bc1c8ec9b0ecfd4a70180b0340f43bfb3b07f35febdba2be78e0439a76 +size 5948273 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T132817404593Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T132817404593Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc6275d86b96693fbc47e732afcb2dd89e8500d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T132817404593Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 23646179840, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3394145280, + "inactive_split_bytes_peak": 5439753728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:17.404124Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 23646179840, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 79921629184, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 79569307648, + "device": 2, + "inactive_split_bytes_current": 2685307904, + "inactive_split_bytes_peak": 5439753728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 96101990400, + "reserved_bytes_peak": 96101990400, + "segment_count_current": 506, + "segment_count_peak": 506, + "timestamp_utc": "2026-08-01T13:28:16.420004Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3394145280, + "inactive_split_bytes_current_delta": 708837376, + "inactive_split_bytes_peak": 5439753728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 10263461888, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 10263461888, + "segment_count_current": 578, + "segment_count_current_delta": 72, + "segment_count_peak": 578, + "segment_count_peak_delta": 72, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T132817404593Z.pickle", + "timestamp_utc": "2026-08-01T13:28:17.521720Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T132817404593Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T132817404593Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..200d061d325f78df52bf9b3862e7a09f88923c63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T132817404593Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42179c64c5d552c4384ae405be2378696c5ca3a2230b60a930a058b821ccc3e9 +size 5976182 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T132818527022Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T132818527022Z.json new file mode 100644 index 0000000000000000000000000000000000000000..58607c77d05f6773e3bb89ac1874891434cdfefd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T132818527022Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3132001280, + "inactive_split_bytes_peak": 5439753728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:18.526595Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3394145280, + "inactive_split_bytes_peak": 5439753728, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:17.536117Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3132001280, + "inactive_split_bytes_current_delta": -262144000, + "inactive_split_bytes_peak": 5439753728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 578, + "segment_count_current_delta": 0, + "segment_count_peak": 578, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T132818527022Z.pickle", + "timestamp_utc": "2026-08-01T13:28:18.636502Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T132818527022Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T132818527022Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb003389525ef2e54c7f902cdeda7df16dd640ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T132818527022Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a7912e2b79ba2b591c00ae646b19ca7875947f41205fae64a1ec9c581de00e6 +size 6654188 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T132819923772Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T132819923772Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c02ee982396e126877907d867590f2a632fe9723 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T132819923772Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5439753728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:19.923370Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3132001280, + "inactive_split_bytes_peak": 5439753728, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:18.650467Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": -90177536, + "inactive_split_bytes_peak": 5439753728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 578, + "segment_count_current_delta": 0, + "segment_count_peak": 578, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T132819923772Z.pickle", + "timestamp_utc": "2026-08-01T13:28:20.034460Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T132819923772Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T132819923772Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..01c1d3ac0e1d61bf4b5b222f10a05fae01392f47 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T132819923772Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:519cbf25210e9c4deee07354723ff5750a4db33ef10c66b4dc92f0dbf317f1d0 +size 6653301 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T132821307921Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T132821307921Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d3ce8feff617f6bf5d8aa95c3c8265e262bd0070 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T132821307921Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5439753728, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:21.307530Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5439753728, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:20.049218Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5439753728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 578, + "segment_count_current_delta": 0, + "segment_count_peak": 578, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T132821307921Z.pickle", + "timestamp_utc": "2026-08-01T13:28:21.419781Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T132821307921Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T132821307921Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..15a5e92e2c0a14af777508d9d15a1761a84c1ed6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T132821307921Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6851ee6434d420ca050b01e492e8fdaa89ba0711f7598f2764d2b044c61b9fb8 +size 6653202 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T132822447986Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T132822447986Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0fd4d35bcaefacb8454569a3c22e98e5b6902e44 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T132822447986Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5439753728, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:22.447589Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5439753728, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:21.433739Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5439753728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 578, + "segment_count_current_delta": 0, + "segment_count_peak": 578, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T132822447986Z.pickle", + "timestamp_utc": "2026-08-01T13:28:22.560360Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T132822447986Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T132822447986Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..700a003ea5425c190a504e863e609f4fabe8caf2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T132822447986Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3104df3c05bf0e53cb75a0178cf2b1a5c8728d4b678e2bbfa9946770d5e8a0c2 +size 6653196 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T132823575804Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T132823575804Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf04b4d1b42befdd475e3afea29b9280f2c626b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T132823575804Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3276704768, + "inactive_split_bytes_peak": 5439753728, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:23.575400Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5439753728, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:22.577017Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3276704768, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 5439753728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 578, + "segment_count_current_delta": 0, + "segment_count_peak": 578, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T132823575804Z.pickle", + "timestamp_utc": "2026-08-01T13:28:23.682169Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T132823575804Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T132823575804Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..484ae787f2501eeeb61ca396b6818a3c218dab85 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T132823575804Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44b560ad14aba31f09579f734fe490f265bea8286c040545dd9fe444f51ad606 +size 6653239 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T132824879775Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T132824879775Z.json new file mode 100644 index 0000000000000000000000000000000000000000..74aa27f87cf4e0c76dbab28f7256cbcfa5909fe4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T132824879775Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103915929600, + "active_bytes_peak_delta": 348120576, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103915929600, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5439753728, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:24.879373Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3276704768, + "inactive_split_bytes_peak": 5439753728, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:23.695496Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 5439753728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 578, + "segment_count_current_delta": 0, + "segment_count_peak": 578, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T132824879775Z.pickle", + "timestamp_utc": "2026-08-01T13:28:24.986871Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T132824879775Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T132824879775Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3d9115eb9e2d3b3cfc2f8f862685ed0306f98b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T132824879775Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4dd0132fad1518fffe6ebb036116d7baab900b94bba4c0d92fb3da85b85a94e +size 6653230 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T132826001218Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T132826001218Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a532b4a5e9c36b8820f9eb66aadd78225272587 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T132826001218Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103915929600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103915929600, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5439753728, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:26.000815Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103915929600, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5439753728, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:25.000182Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5439753728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 578, + "segment_count_current_delta": 0, + "segment_count_peak": 578, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T132826001218Z.pickle", + "timestamp_utc": "2026-08-01T13:28:26.105770Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T132826001218Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T132826001218Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..43ba8f27954fb3087ff50ad2bf04f3384ee31347 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T132826001218Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32182ee35f9bbff84d97f3b70405e20dc599b5f99e5d9e9fba9ecf13628ddd9d +size 6653230 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T132921090513Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T132921090513Z.json new file mode 100644 index 0000000000000000000000000000000000000000..13418cf7766cf0ceacb5c8360b598be0033bd584 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T132921090513Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166679040, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103915929600, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166679040, + "active_bytes_peak": 103915929600, + "allocated_bytes_current": 61166679040, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3389950976, + "inactive_split_bytes_peak": 5439753728, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:29:21.090034Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166679040, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103915929600, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 2, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5439753728, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 578, + "segment_count_peak": 578, + "timestamp_utc": "2026-08-01T13:28:26.118951Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3389950976, + "inactive_split_bytes_current_delta": 348127232, + "inactive_split_bytes_peak": 5439753728, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 578, + "segment_count_current_delta": 0, + "segment_count_peak": 578, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T132921090513Z.pickle", + "timestamp_utc": "2026-08-01T13:29:21.295249Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T132921090513Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T132921090513Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..28cefdf55efcfe71bb1fcdb79e8c5b0426d162bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T132921090513Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5529ebfaccde5cc57ee3dc7366e68c2767db9d8565998c3db940895ffc5c4b8e +size 10548400 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T132816253275Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T132816253275Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b6f0c60506fe6cdc42928fe68a48184d80eef703 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T132816253275Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 37209512960, + "active_bytes_peak": 79923234304, + "active_bytes_peak_delta": 44456898048, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 79923234304, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 79570912768, + "device": 3, + "inactive_split_bytes_current": 2450426880, + "inactive_split_bytes_peak": 5324672512, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 96101990400, + "reserved_bytes_peak": 96101990400, + "segment_count_current": 511, + "segment_count_peak": 511, + "timestamp_utc": "2026-08-01T13:28:16.252657Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 37209512960, + "allocated_bytes_peak": 79570912768, + "allocated_bytes_peak_delta": 44104576512, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:27:07.177926Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2450426880, + "inactive_split_bytes_current_delta": 2317607936, + "inactive_split_bytes_peak": 5324672512, + "inactive_split_bytes_peak_delta": 5164286464, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 96101990400, + "reserved_bytes_current_delta": 56857985024, + "reserved_bytes_peak": 96101990400, + "reserved_bytes_peak_delta": 56857985024, + "segment_count_current": 511, + "segment_count_current_delta": 440, + "segment_count_peak": 511, + "segment_count_peak_delta": 440, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T132816253275Z.pickle", + "timestamp_utc": "2026-08-01T13:28:16.372316Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T132816253275Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T132816253275Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ce639564f277e1c94218adc88348a566d4fa0b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T132816253275Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2f9f135271d5c8a800b8d7c69f8df24db0b4df53b846955cd1638c5b482645d +size 5948953 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T132817404399Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T132817404399Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7cdf24920279757b77353969c7486e0842150862 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T132817404399Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103559985664, + "active_bytes_peak_delta": 23636751360, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103559985664, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 2328792064, + "inactive_split_bytes_peak": 5324672512, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:17.403976Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 23643546112, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 79923234304, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 79570912768, + "device": 3, + "inactive_split_bytes_current": 2450426880, + "inactive_split_bytes_peak": 5324672512, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 96101990400, + "reserved_bytes_peak": 96101990400, + "segment_count_current": 511, + "segment_count_peak": 511, + "timestamp_utc": "2026-08-01T13:28:16.388357Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 2328792064, + "inactive_split_bytes_current_delta": -121634816, + "inactive_split_bytes_peak": 5324672512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 10261364736, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 10261364736, + "segment_count_current": 580, + "segment_count_current_delta": 69, + "segment_count_peak": 580, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T132817404399Z.pickle", + "timestamp_utc": "2026-08-01T13:28:17.521122Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T132817404399Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T132817404399Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0d5d6e477042074724fd1ae5d0128988eafc460d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T132817404399Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f305862d51eea21b1ffdf92d89bda562f2924676e7ab3d62ba7a042b8c955cc7 +size 5976131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T132818526994Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T132818526994Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f844e9b5561085a9412549d79b63f7e358e72876 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T132818526994Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564167168, + "active_bytes_peak_delta": 4181504, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3276704768, + "inactive_split_bytes_peak": 5324672512, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:18.526593Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103559985664, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 2328792064, + "inactive_split_bytes_peak": 5324672512, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:17.535101Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3276704768, + "inactive_split_bytes_current_delta": 947912704, + "inactive_split_bytes_peak": 5324672512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T132818526994Z.pickle", + "timestamp_utc": "2026-08-01T13:28:18.639112Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T132818526994Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T132818526994Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd11f970e2030ad338d6a5d4b65bb46951d19f73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T132818526994Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db46f268c8497130dc5cba01020cfc30d0e73cb3c9d2b9225c87f957002d3ead +size 6654019 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T132819923749Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T132819923749Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f0541088e976530cef19617a7ade8c19c730526 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T132819923749Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564167168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5324672512, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:19.923354Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3276704768, + "inactive_split_bytes_peak": 5324672512, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:18.652525Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 5324672512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T132819923749Z.pickle", + "timestamp_utc": "2026-08-01T13:28:20.037273Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T132819923749Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T132819923749Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4206d3d0593cc00a18ceea9d3f9b4d1102b82d6d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T132819923749Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85b274ac1e442f3b7ea706331a46f63869647dbfa61888bd2851953026f36c9e +size 6654012 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T132821307928Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T132821307928Z.json new file mode 100644 index 0000000000000000000000000000000000000000..54b1ff3f7179057b2f5f950570fc81863d2db887 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T132821307928Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564167168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5324672512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:21.307530Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5324672512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:20.053626Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5324672512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T132821307928Z.pickle", + "timestamp_utc": "2026-08-01T13:28:21.417303Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T132821307928Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T132821307928Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6cf1c8060f10ca2a10ee552ab8fa9ed87cd30840 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T132821307928Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eb3a0d1586b1f7db7a69eb78260786f097f438e0043b1b71bcade58150450d2 +size 6654012 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T132822447995Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T132822447995Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3494b58c375a41e0035e609988aeae14893d3af9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T132822447995Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564167168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5324672512, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:22.447592Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5324672512, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:21.430908Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5324672512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T132822447995Z.pickle", + "timestamp_utc": "2026-08-01T13:28:22.554933Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T132822447995Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T132822447995Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9bca1fc98697797636ea432225693d2219b68b24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T132822447995Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ffba839a9c6d11079835101d42faaf9f2e946666958dfa51438562cc538095b +size 6653843 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T132823575797Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T132823575797Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7a34cd412fe9ca24d72d36ec29af02f1828ad73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T132823575797Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564167168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5324672512, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:23.575394Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5324672512, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:22.568456Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5324672512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T132823575797Z.pickle", + "timestamp_utc": "2026-08-01T13:28:23.684058Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T132823575797Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T132823575797Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd274b70b46951b97f56c20df74156b5cf5778f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T132823575797Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:151e5172ddd9f777485c7090d9f368cb159d5df80afe85c98822546d0448e074 +size 6653844 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T132824879843Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T132824879843Z.json new file mode 100644 index 0000000000000000000000000000000000000000..010245c8b6e2c1a2158aa1514897541d91f25650 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T132824879843Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564167168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5324672512, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:24.879410Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5324672512, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:23.697597Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5324672512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T132824879843Z.pickle", + "timestamp_utc": "2026-08-01T13:28:24.987645Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T132824879843Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T132824879843Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a94108e94844d8d4a84551026f9536818a701069 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T132824879843Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b5d5b9add0dddd83e32b35c118ce84ece2e3523b88fcef6a8254ee2df268baf +size 6653844 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T132826001232Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T132826001232Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f7bb95cc819703f3ee83cbe315547827503e604 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T132826001232Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564167168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5324672512, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:26.000827Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5324672512, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:25.001131Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5324672512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T132826001232Z.pickle", + "timestamp_utc": "2026-08-01T13:28:26.107626Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T132826001232Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T132826001232Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0f8e71dbc9394e76813c5f3da95780967c9fb1bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T132826001232Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20af69e1b58bec084a7433cea6852ac00e58fb08456fe4f9792dcfc7861734db +size 6653844 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T132921090511Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T132921090511Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c6bf1834ef96c14b354bc9439e9c2c4835da44d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T132921090511Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166679040, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103564167168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166679040, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 61166679040, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3033435136, + "inactive_split_bytes_peak": 5324672512, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:29:21.090026Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166679040, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103564167168, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 3, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5324672512, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:26.121055Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3033435136, + "inactive_split_bytes_current_delta": -8388608, + "inactive_split_bytes_peak": 5324672512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T132921090511Z.pickle", + "timestamp_utc": "2026-08-01T13:29:21.295586Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T132921090511Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T132921090511Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..927b3862dca66d78ce2db5be9d67cc8189374e85 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T132921090511Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c20cca471921aa281281e92ffe6d92c8c76e90268d98138159d3ac2eeb2e694c +size 10548931 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T132816253204Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T132816253204Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bed4a1e8ce5bb2f843582c93f701355198221677 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T132816253204Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442265600, + "active_bytes_current_delta": 37210368000, + "active_bytes_peak": 80265954304, + "active_bytes_peak_delta": 44799175680, + "after": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 80265954304, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 79565244416, + "device": 4, + "inactive_split_bytes_current": 2923085824, + "inactive_split_bytes_peak": 5554986496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 96571752448, + "reserved_bytes_peak": 96571752448, + "segment_count_current": 507, + "segment_count_peak": 507, + "timestamp_utc": "2026-08-01T13:28:16.252638Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442265600, + "allocated_bytes_current_delta": 37210368000, + "allocated_bytes_peak": 79565244416, + "allocated_bytes_peak_delta": 44098465792, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:27:07.177476Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 2923085824, + "inactive_split_bytes_current_delta": 2790709248, + "inactive_split_bytes_peak": 5554986496, + "inactive_split_bytes_peak_delta": 5401576960, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 96571752448, + "reserved_bytes_current_delta": 57327747072, + "reserved_bytes_peak": 96571752448, + "reserved_bytes_peak_delta": 57327747072, + "segment_count_current": 507, + "segment_count_current_delta": 436, + "segment_count_peak": 507, + "segment_count_peak_delta": 436, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T132816253204Z.pickle", + "timestamp_utc": "2026-08-01T13:28:16.370685Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T132816253204Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T132816253204Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c6da776b923212441c7344d5c4bfb3a972b6e763 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T132816253204Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3d4123ae483b45d8ff5a5d3cd230c5a9a179078462af7b1085b9119ff607dd3 +size 5957248 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T132817404355Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T132817404355Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76f0f556e578012829b0a74d3c7903231d423119 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T132817404355Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442265600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103555954688, + "active_bytes_peak_delta": 23290000384, + "after": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103555954688, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103203633152, + "device": 4, + "inactive_split_bytes_current": 3514482688, + "inactive_split_bytes_peak": 5554986496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:17.403953Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442265600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103203633152, + "allocated_bytes_peak_delta": 23638388736, + "before": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 80265954304, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 79565244416, + "device": 4, + "inactive_split_bytes_current": 2923085824, + "inactive_split_bytes_peak": 5554986496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 96571752448, + "reserved_bytes_peak": 96571752448, + "segment_count_current": 507, + "segment_count_peak": 507, + "timestamp_utc": "2026-08-01T13:28:16.387252Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3514482688, + "inactive_split_bytes_current_delta": 591396864, + "inactive_split_bytes_peak": 5554986496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106375938048, + "reserved_bytes_current_delta": 9804185600, + "reserved_bytes_peak": 106375938048, + "reserved_bytes_peak_delta": 9804185600, + "segment_count_current": 577, + "segment_count_current_delta": 70, + "segment_count_peak": 577, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T132817404355Z.pickle", + "timestamp_utc": "2026-08-01T13:28:17.519396Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T132817404355Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T132817404355Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..77c77935935308a978df2243cb0b554bdfad7f6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T132817404355Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39032dea8bac7de3ba8b2522594fac405966112869aaf24edb3b15b5851b62a8 +size 6661350 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T132818526957Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T132818526957Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e9d5e222c5a8388d8ff6107a331646c5feec542d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T132818526957Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442265600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103561132032, + "active_bytes_peak_delta": 5177344, + "after": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3340419072, + "inactive_split_bytes_peak": 5554986496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:18.526566Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442265600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103208810496, + "allocated_bytes_peak_delta": 5177344, + "before": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103555954688, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103203633152, + "device": 4, + "inactive_split_bytes_current": 3514482688, + "inactive_split_bytes_peak": 5554986496, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:17.533728Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3340419072, + "inactive_split_bytes_current_delta": -174063616, + "inactive_split_bytes_peak": 5554986496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106375938048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106375938048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 577, + "segment_count_current_delta": 0, + "segment_count_peak": 577, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T132818526957Z.pickle", + "timestamp_utc": "2026-08-01T13:28:18.634252Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T132818526957Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T132818526957Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..43eb4201a8c2360e3256d1c623907649d84774f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T132818526957Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:984a5394bd851976c0dc74a28a40e2b74252f44f375364ee389f70c537efe11b +size 6662047 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T132819923720Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T132819923720Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76e3ce64be90ee4ecf60e046d8092078089e0d92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T132819923720Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442265600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103561132032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3101343744, + "inactive_split_bytes_peak": 5554986496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:19.923319Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442265600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103208810496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3340419072, + "inactive_split_bytes_peak": 5554986496, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:18.647941Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3101343744, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 5554986496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106375938048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106375938048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 577, + "segment_count_current_delta": 0, + "segment_count_peak": 577, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T132819923720Z.pickle", + "timestamp_utc": "2026-08-01T13:28:20.033942Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T132819923720Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T132819923720Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9200d023b259fd5f920eed690d53db8703484732 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T132819923720Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26cf4297fe5aef61c43c7438bde5799e22ec9c36c41e4eab7f15c25b6629e4b5 +size 6660846 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T132821307935Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T132821307935Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b49084846794ab07e4a2fd301e2949f7d5cef474 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T132821307935Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442265600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103561132032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3575300096, + "inactive_split_bytes_peak": 5554986496, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:21.307515Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442265600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103208810496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3101343744, + "inactive_split_bytes_peak": 5554986496, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:20.048224Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3575300096, + "inactive_split_bytes_current_delta": 473956352, + "inactive_split_bytes_peak": 5554986496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106375938048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106375938048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 577, + "segment_count_current_delta": 0, + "segment_count_peak": 577, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T132821307935Z.pickle", + "timestamp_utc": "2026-08-01T13:28:21.422823Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T132821307935Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T132821307935Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c8644947566ff0a699228fa7a3dab37b178e596 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T132821307935Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:624505066d9b0421e880e3c026802e3b1f02ba11b9b190d0942c0d5eeb9b66e2 +size 6660925 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T132822447999Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T132822447999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6498fe79f5286b0befb5d3ed5e1de013389df105 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T132822447999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442265600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103561132032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3017457664, + "inactive_split_bytes_peak": 5554986496, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:22.447611Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442265600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103208810496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3575300096, + "inactive_split_bytes_peak": 5554986496, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:21.438753Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3017457664, + "inactive_split_bytes_current_delta": -557842432, + "inactive_split_bytes_peak": 5554986496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106375938048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106375938048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 577, + "segment_count_current_delta": 0, + "segment_count_peak": 577, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T132822447999Z.pickle", + "timestamp_utc": "2026-08-01T13:28:22.553074Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T132822447999Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T132822447999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6c38465f5124be9dd7bdb31ae37ebf8e25046cf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T132822447999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38bb4ce93be3975ecb2b1917b7433258879be752991892d988fae638226ac451 +size 6660577 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T132823575793Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T132823575793Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5631c7d10604e7c0e1d82ddb148c25cd7c3658c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T132823575793Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442265600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103561132032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3340419072, + "inactive_split_bytes_peak": 5554986496, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:23.575392Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442265600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103208810496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3017457664, + "inactive_split_bytes_peak": 5554986496, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:22.566157Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3340419072, + "inactive_split_bytes_current_delta": 322961408, + "inactive_split_bytes_peak": 5554986496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106375938048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106375938048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 577, + "segment_count_current_delta": 0, + "segment_count_peak": 577, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T132823575793Z.pickle", + "timestamp_utc": "2026-08-01T13:28:23.679855Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T132823575793Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T132823575793Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6630f67a34c5f6d525fd56f5ceb3c66e5a8db22a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T132823575793Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afc7de47d51a39ebe96cf58dacf3220fc2735d72604b7ba9d662293163afd537 +size 6660570 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T132824879770Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T132824879770Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5018dfdcea25fa8e906a263b84aafe5dd6524d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T132824879770Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442265600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103561132032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3548037120, + "inactive_split_bytes_peak": 5554986496, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:24.879362Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442265600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103208810496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3340419072, + "inactive_split_bytes_peak": 5554986496, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:23.693193Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3548037120, + "inactive_split_bytes_current_delta": 207618048, + "inactive_split_bytes_peak": 5554986496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106375938048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106375938048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 577, + "segment_count_current_delta": 0, + "segment_count_peak": 577, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T132824879770Z.pickle", + "timestamp_utc": "2026-08-01T13:28:24.984185Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T132824879770Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T132824879770Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..287217ac5dac8b8cef979a918135ad0ff8594ffb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T132824879770Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78fecb8ec73c9a837df72e61fc787978952ffa74002a7994c0f4b4a03a08549b +size 6660582 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T132826001215Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T132826001215Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c685f2b4a16603f86777af79e3ccf266f81d19a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T132826001215Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442265600, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103561132032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3575300096, + "inactive_split_bytes_peak": 5554986496, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:26.000805Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442265600, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103208810496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3548037120, + "inactive_split_bytes_peak": 5554986496, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:24.997427Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3575300096, + "inactive_split_bytes_current_delta": 27262976, + "inactive_split_bytes_peak": 5554986496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106375938048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106375938048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 577, + "segment_count_current_delta": 0, + "segment_count_peak": 577, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T132826001215Z.pickle", + "timestamp_utc": "2026-08-01T13:28:26.104447Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T132826001215Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T132826001215Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4cd7f189532479ab0665e8beadf7466636ae4e0f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T132826001215Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:776716064e9b20f1398ca9a62bba5b358321e0c515e90691f9161414c4a88d44 +size 6660581 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T132921090505Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T132921090505Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d8f7449f604959822faad7dc880e178dc7625ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T132921090505Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61167976448, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103561132032, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61167976448, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 61167976448, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3684352000, + "inactive_split_bytes_peak": 5554986496, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:29:21.090029Z", + "world_size": 8 + }, + "allocated_bytes_current": 61167976448, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103208810496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442265600, + "active_bytes_peak": 103561132032, + "allocated_bytes_current": 72442265600, + "allocated_bytes_peak": 103208810496, + "device": 4, + "inactive_split_bytes_current": 3575300096, + "inactive_split_bytes_peak": 5554986496, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106375938048, + "reserved_bytes_peak": 106375938048, + "segment_count_current": 577, + "segment_count_peak": 577, + "timestamp_utc": "2026-08-01T13:28:26.117989Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3684352000, + "inactive_split_bytes_current_delta": 109051904, + "inactive_split_bytes_peak": 5554986496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106375938048, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106375938048, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 577, + "segment_count_current_delta": 0, + "segment_count_peak": 577, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T132921090505Z.pickle", + "timestamp_utc": "2026-08-01T13:29:21.291667Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T132921090505Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T132921090505Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c7136b7ae9a19f4bee0066350ae55d469b2f1c95 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T132921090505Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ccc229b0aaf3e1ca8d95cf8b9743b6d0e8d3f9c8112cf9eb12eb6e97b461b97 +size 10558090 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T132816253241Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T132816253241Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fdf69e7bd0b5aec460686f664d5b9466bc51f5f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T132816253241Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 37209512960, + "active_bytes_peak": 79921629184, + "active_bytes_peak_delta": 44455292928, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 79921629184, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 79569307648, + "device": 5, + "inactive_split_bytes_current": 2802748416, + "inactive_split_bytes_peak": 5322313216, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 96336871424, + "reserved_bytes_peak": 96336871424, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:28:16.252629Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 37209512960, + "allocated_bytes_peak": 79569307648, + "allocated_bytes_peak_delta": 44102971392, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:27:07.177970Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 2802748416, + "inactive_split_bytes_current_delta": 2669929472, + "inactive_split_bytes_peak": 5322313216, + "inactive_split_bytes_peak_delta": 5161927168, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 96336871424, + "reserved_bytes_current_delta": 57092866048, + "reserved_bytes_peak": 96336871424, + "reserved_bytes_peak_delta": 57092866048, + "segment_count_current": 505, + "segment_count_current_delta": 434, + "segment_count_peak": 505, + "segment_count_peak_delta": 434, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T132816253241Z.pickle", + "timestamp_utc": "2026-08-01T13:28:16.371217Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T132816253241Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T132816253241Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d518502700c64e2a7bc5f14d2aad3e3c6924e347 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T132816253241Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a2a5fca2e0b2e124aca3f0034e68dd6b3ce815dbbfaaa8cdaca04d761f1ba8e +size 5948187 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T132817404398Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T132817404398Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96294b5c792bfb44338735069ceeed949df04f73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T132817404398Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 23646179840, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3629026304, + "inactive_split_bytes_peak": 5322313216, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:17.403961Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 23646179840, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 79921629184, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 79569307648, + "device": 5, + "inactive_split_bytes_current": 2802748416, + "inactive_split_bytes_peak": 5322313216, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 96336871424, + "reserved_bytes_peak": 96336871424, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:28:16.390416Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3629026304, + "inactive_split_bytes_current_delta": 826277888, + "inactive_split_bytes_peak": 5322313216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 10028580864, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 10028580864, + "segment_count_current": 573, + "segment_count_current_delta": 68, + "segment_count_peak": 573, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T132817404398Z.pickle", + "timestamp_utc": "2026-08-01T13:28:17.522973Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T132817404398Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T132817404398Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db4d4c46942074a8f44b2fe9ad080b70b30671a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T132817404398Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14ee3dab69da22c9f74fe2df8f9a2f87d2089b9a7a6db91fd75eafc3d98f1823 +size 5975088 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T132818526949Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T132818526949Z.json new file mode 100644 index 0000000000000000000000000000000000000000..303a10ec7ac2c529798303cc51aa7176011b24b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T132818526949Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3689843712, + "inactive_split_bytes_peak": 5322313216, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:18.526548Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3629026304, + "inactive_split_bytes_peak": 5322313216, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:17.537533Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3689843712, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 5322313216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 573, + "segment_count_current_delta": 0, + "segment_count_peak": 573, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T132818526949Z.pickle", + "timestamp_utc": "2026-08-01T13:28:18.633300Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T132818526949Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T132818526949Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8f4543a8b6aa81db0519af494cd5c087ddaa6c73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T132818526949Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f2d39e4a21f5434d86cd589cbb1a18164885ab90a7db79b57e3a6d04f8b8daa +size 5972082 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T132819923715Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T132819923715Z.json new file mode 100644 index 0000000000000000000000000000000000000000..90ac34c76aaf6bf752b90642b3f4b90b4762f9b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T132819923715Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3689843712, + "inactive_split_bytes_peak": 5322313216, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:19.923312Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3689843712, + "inactive_split_bytes_peak": 5322313216, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:18.647018Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3689843712, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5322313216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 573, + "segment_count_current_delta": 0, + "segment_count_peak": 573, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T132819923715Z.pickle", + "timestamp_utc": "2026-08-01T13:28:20.035472Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T132819923715Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T132819923715Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..52cc075cc5f81b34bcefdcadfbe8e86cb571c3a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T132819923715Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:729936ffa36b381d05c5974ed421b10663fb831c7cf7a195fc11c100dea4bf06 +size 5971962 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T132821307902Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T132821307902Z.json new file mode 100644 index 0000000000000000000000000000000000000000..044bd0056c4f7a65d10fbf146bfe0d41f9a77fb0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T132821307902Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3454962688, + "inactive_split_bytes_peak": 5322313216, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:21.307497Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3689843712, + "inactive_split_bytes_peak": 5322313216, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:20.052022Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3454962688, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 5322313216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 573, + "segment_count_current_delta": 0, + "segment_count_peak": 573, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T132821307902Z.pickle", + "timestamp_utc": "2026-08-01T13:28:21.418233Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T132821307902Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T132821307902Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..330410488bb4088939914ae4551dd8a8a5558c2c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T132821307902Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a415a91df438ed0247b66708fcafba80463ed59b9bf652cf3b1fcf41a7a701dc +size 5971812 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T132822447949Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T132822447949Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f1eac7c68f963fefd6b80c321ca3f3478f63205 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T132822447949Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3454962688, + "inactive_split_bytes_peak": 5322313216, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:22.447548Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3454962688, + "inactive_split_bytes_peak": 5322313216, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:21.431955Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3454962688, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5322313216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 573, + "segment_count_current_delta": 0, + "segment_count_peak": 573, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T132822447949Z.pickle", + "timestamp_utc": "2026-08-01T13:28:22.556450Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T132822447949Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T132822447949Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a85b8480c83e094fead5f801d0c10d898f76047 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T132822447949Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:470feb52ea22265fe5805a0e3bb1c954ea35168d097873b39b23691b5e373247 +size 5971729 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T132823575808Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T132823575808Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a00ef54cc2ad27444dfb125aee7898ca3941424 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T132823575808Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3276704768, + "inactive_split_bytes_peak": 5322313216, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:23.575384Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3454962688, + "inactive_split_bytes_peak": 5322313216, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:22.570116Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3276704768, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 5322313216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 573, + "segment_count_current_delta": 0, + "segment_count_peak": 573, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T132823575808Z.pickle", + "timestamp_utc": "2026-08-01T13:28:23.678512Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T132823575808Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T132823575808Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..68f57a9386f1baaa6224c4d95f7c0d32694edf41 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T132823575808Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9c46017f810a58c07f59e417fa5c4841044d2ef526d67183d2282161f16180d +size 5971680 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T132824879743Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T132824879743Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f65b0f3f1784a057895e2715fc426530338a94b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T132824879743Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3454962688, + "inactive_split_bytes_peak": 5322313216, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:24.879351Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3276704768, + "inactive_split_bytes_peak": 5322313216, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:23.692221Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3454962688, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 5322313216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 573, + "segment_count_current_delta": 0, + "segment_count_peak": 573, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T132824879743Z.pickle", + "timestamp_utc": "2026-08-01T13:28:24.989219Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T132824879743Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T132824879743Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ce70c65f85cea1374f5af07f64f7ef1ee6ae828 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T132824879743Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:401e7e6ddb85438bdc3534fe5935c27823ff867d1587e695780636865abbc538 +size 5971769 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T132826001193Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T132826001193Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ac32d1f14f53091bb4b81e76b21234bba275ae5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T132826001193Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3454962688, + "inactive_split_bytes_peak": 5322313216, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:26.000793Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3454962688, + "inactive_split_bytes_peak": 5322313216, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:25.005301Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3454962688, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5322313216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 573, + "segment_count_current_delta": 0, + "segment_count_peak": 573, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T132826001193Z.pickle", + "timestamp_utc": "2026-08-01T13:28:26.103439Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T132826001193Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T132826001193Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7346879dc963dafe20391b54e20012cac71be8f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T132826001193Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ffefb6ff6e6451c76239ad8ffee0f716e964eef22fb238ab3258ca049c9b43b +size 5971761 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T132921090613Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T132921090613Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6df9b0bd2443d2cda0cad0e6419f9f04e6de438 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T132921090613Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166679040, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103567809024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166679040, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 61166679040, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3859713024, + "inactive_split_bytes_peak": 5322313216, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:29:21.090028Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166679040, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103215487488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103567809024, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103215487488, + "device": 5, + "inactive_split_bytes_current": 3454962688, + "inactive_split_bytes_peak": 5322313216, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106365452288, + "reserved_bytes_peak": 106365452288, + "segment_count_current": 573, + "segment_count_peak": 573, + "timestamp_utc": "2026-08-01T13:28:26.116815Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3859713024, + "inactive_split_bytes_current_delta": 404750336, + "inactive_split_bytes_peak": 5322313216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106365452288, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106365452288, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 573, + "segment_count_current_delta": 0, + "segment_count_peak": 573, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T132921090613Z.pickle", + "timestamp_utc": "2026-08-01T13:29:21.310292Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T132921090613Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T132921090613Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f9810825bfc032093df1c87f2eed277202231b07 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T132921090613Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa882af939424c672ff49e5e0b287c39f051b474e87eb53352930179007ea640 +size 9735930 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T132816253329Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T132816253329Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e87c898d3127ec020c745d7c361f933eabd2cc5b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T132816253329Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 37209512960, + "active_bytes_peak": 79923234304, + "active_bytes_peak_delta": 44456898048, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 79923234304, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 79570912768, + "device": 6, + "inactive_split_bytes_current": 2567867392, + "inactive_split_bytes_peak": 5794434560, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 96336871424, + "reserved_bytes_peak": 96336871424, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:28:16.252691Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 37209512960, + "allocated_bytes_peak": 79570912768, + "allocated_bytes_peak_delta": 44104576512, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:27:07.177909Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2567867392, + "inactive_split_bytes_current_delta": 2435048448, + "inactive_split_bytes_peak": 5794434560, + "inactive_split_bytes_peak_delta": 5634048512, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 96336871424, + "reserved_bytes_current_delta": 57092866048, + "reserved_bytes_peak": 96336871424, + "reserved_bytes_peak_delta": 57092866048, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T132816253329Z.pickle", + "timestamp_utc": "2026-08-01T13:28:16.373119Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T132816253329Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T132816253329Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c11ade499091b70691e53162e6d5e50e1545b2c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T132816253329Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02585b1e72cd903807d37b5decd6743ae3658893758ff1a31dc56dd921322910 +size 5949150 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T132817404384Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T132817404384Z.json new file mode 100644 index 0000000000000000000000000000000000000000..255b062e62b8f14cb7bf07b4f2469131702bb472 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T132817404384Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103566780416, + "active_bytes_peak_delta": 23643546112, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 3098446848, + "inactive_split_bytes_peak": 5794434560, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:17.403973Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 23643546112, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 79923234304, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 79570912768, + "device": 6, + "inactive_split_bytes_current": 2567867392, + "inactive_split_bytes_peak": 5794434560, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 96336871424, + "reserved_bytes_peak": 96336871424, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:28:16.391508Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3098446848, + "inactive_split_bytes_current_delta": 530579456, + "inactive_split_bytes_peak": 5794434560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 10026483712, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 10026483712, + "segment_count_current": 581, + "segment_count_current_delta": 69, + "segment_count_peak": 581, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T132817404384Z.pickle", + "timestamp_utc": "2026-08-01T13:28:17.519782Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T132817404384Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T132817404384Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a78cc7854a41b7c8df7b68b73cfe6f915742939 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T132817404384Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:339a3ccfed59df272e9c7ddbd1630da976d0d31b743c5b592995c1df4eaf8557 +size 5976388 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T132818526978Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T132818526978Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e7c0ff260c97cad8191a4619d449f9cb0dafba3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T132818526978Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103566780416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 2863565824, + "inactive_split_bytes_peak": 5794434560, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:18.526568Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 3098446848, + "inactive_split_bytes_peak": 5794434560, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:17.534117Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2863565824, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 5794434560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 581, + "segment_count_current_delta": 0, + "segment_count_peak": 581, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T132818526978Z.pickle", + "timestamp_utc": "2026-08-01T13:28:18.634379Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T132818526978Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T132818526978Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..50683d6da99fa9e14edeab074ca22ffb2db06bfe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T132818526978Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef1b81708f89008d340dae8b1a6a5ce2b23e0b55aac5f76da99fb658ca622f5c +size 5973347 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T132819923729Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T132819923729Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d2e38867023eca24b13cc786ddaec68f7ca809f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T132819923729Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103566780416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 2863565824, + "inactive_split_bytes_peak": 5794434560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:19.923317Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 2863565824, + "inactive_split_bytes_peak": 5794434560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:18.648184Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2863565824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5794434560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 581, + "segment_count_current_delta": 0, + "segment_count_peak": 581, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T132819923729Z.pickle", + "timestamp_utc": "2026-08-01T13:28:20.033946Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T132819923729Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T132819923729Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f88b756bb38929e412391f01de97a689998234d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T132819923729Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0564c2074395610b3ffcb399eca22e4a404f4f2e9a27ea5d2e0b0d759c950953 +size 5973352 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T132821307918Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T132821307918Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a5a67d9fba543577a9115d955f4bbf1fa1615fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T132821307918Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103566780416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 2863565824, + "inactive_split_bytes_peak": 5794434560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:21.307511Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 2863565824, + "inactive_split_bytes_peak": 5794434560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:20.048269Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2863565824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5794434560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 581, + "segment_count_current_delta": 0, + "segment_count_peak": 581, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T132821307918Z.pickle", + "timestamp_utc": "2026-08-01T13:28:21.421593Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T132821307918Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T132821307918Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea539b0221e296fc18fdd8c14f850bb92e487ac9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T132821307918Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d5db419c6fef1651cb3b09fe80e84bf4107a4b00b9d8a4ebe4fda94d8835c53 +size 5973352 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T132822447956Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T132822447956Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ecc5c752de04fa0a6d63aac59d61998c8d63be9a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T132822447956Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103566780416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5794434560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:22.447565Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 2863565824, + "inactive_split_bytes_peak": 5794434560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:21.437778Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 5794434560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 581, + "segment_count_current_delta": 0, + "segment_count_peak": 581, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T132822447956Z.pickle", + "timestamp_utc": "2026-08-01T13:28:22.556831Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T132822447956Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T132822447956Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4d954332741ca5bc399b924ac83da8357d055896 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T132822447956Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56aae899a193b7977ab43a86ef83e097626fea89181d062b32457a14b301c835 +size 5973388 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T132823575788Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T132823575788Z.json new file mode 100644 index 0000000000000000000000000000000000000000..276fd038096e9c8fe5e91037bd765e21b7a1acd6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T132823575788Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103566780416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5794434560, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:23.575379Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5794434560, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:22.570335Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 5794434560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 581, + "segment_count_current_delta": 0, + "segment_count_peak": 581, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T132823575788Z.pickle", + "timestamp_utc": "2026-08-01T13:28:23.680034Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T132823575788Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T132823575788Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e36db9191c31e9ec86f651458a645be8c108919 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T132823575788Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d5b5c278d44fea99cda403ad17d1e3beaf98e2ffe2fe3f742432b26535119e8 +size 5973385 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T132824879791Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T132824879791Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2095d6f8775abdc7a2f4b75135942e70b3bc8acf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T132824879791Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103566780416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 2746125312, + "inactive_split_bytes_peak": 5794434560, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:24.879360Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 3041823744, + "inactive_split_bytes_peak": 5794434560, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:23.693288Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2746125312, + "inactive_split_bytes_current_delta": -295698432, + "inactive_split_bytes_peak": 5794434560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 581, + "segment_count_current_delta": 0, + "segment_count_peak": 581, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T132824879791Z.pickle", + "timestamp_utc": "2026-08-01T13:28:24.987000Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T132824879791Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T132824879791Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a6879b01ef260f926943a9b084cdb5f1937c8cd4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T132824879791Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b006cb123b2785689e4b478ce22497b236673ea27f6a69a6a42edc2219e73b84 +size 5972716 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T132826001210Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T132826001210Z.json new file mode 100644 index 0000000000000000000000000000000000000000..17c3efe50b7f809d14f38272386c1bd1ebabdadd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T132826001210Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968192, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103566780416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 2863565824, + "inactive_split_bytes_peak": 5794434560, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:26.000785Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968192, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 2746125312, + "inactive_split_bytes_peak": 5794434560, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:25.000631Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2863565824, + "inactive_split_bytes_current_delta": 117440512, + "inactive_split_bytes_peak": 5794434560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 581, + "segment_count_current_delta": 0, + "segment_count_peak": 581, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T132826001210Z.pickle", + "timestamp_utc": "2026-08-01T13:28:26.107444Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T132826001210Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T132826001210Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72e54a4efb7db2721f976ea61c26b7e4107792d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T132826001210Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ada7f19d7439856f21c59020a9affe49350c1e2f7da58cd17bafbc42407e1805 +size 5972692 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T132921090593Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T132921090593Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b5964e5f131062e9bb4d9046c7804ff6fa019548 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T132921090593Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166679040, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103566780416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166679040, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 61166679040, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 2798554112, + "inactive_split_bytes_peak": 5794434560, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:29:21.090036Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166679040, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103214458880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968192, + "active_bytes_peak": 103566780416, + "allocated_bytes_current": 72440968192, + "allocated_bytes_peak": 103214458880, + "device": 6, + "inactive_split_bytes_current": 2863565824, + "inactive_split_bytes_peak": 5794434560, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 581, + "segment_count_peak": 581, + "timestamp_utc": "2026-08-01T13:28:26.121413Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 2798554112, + "inactive_split_bytes_current_delta": -65011712, + "inactive_split_bytes_peak": 5794434560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 581, + "segment_count_current_delta": 0, + "segment_count_peak": 581, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T132921090593Z.pickle", + "timestamp_utc": "2026-08-01T13:29:21.301700Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T132921090593Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T132921090593Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..75a393045e2a0cc2e881a8f8dfe244e744e43b00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T132921090593Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b5fe267bae5b1b4df7fdc11f8c808ffdda1f3d3c7bb4ef2255022b9bc82bb67 +size 9737338 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T132816253492Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T132816253492Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f21df5caa2346a722a1283547012e9d926cda806 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T132816253492Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968704, + "active_bytes_current_delta": 37209513472, + "active_bytes_peak": 79923234304, + "active_bytes_peak_delta": 44456898048, + "after": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 79923234304, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 79570912768, + "device": 7, + "inactive_split_bytes_current": 2450426368, + "inactive_split_bytes_peak": 4972350976, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 96101990400, + "reserved_bytes_peak": 96101990400, + "segment_count_current": 511, + "segment_count_peak": 511, + "timestamp_utc": "2026-08-01T13:28:16.252895Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968704, + "allocated_bytes_current_delta": 37209513472, + "allocated_bytes_peak": 79570912768, + "allocated_bytes_peak_delta": 44104576512, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:27:07.177942Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2450426368, + "inactive_split_bytes_current_delta": 2317607424, + "inactive_split_bytes_peak": 4972350976, + "inactive_split_bytes_peak_delta": 4811964928, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 96101990400, + "reserved_bytes_current_delta": 56857985024, + "reserved_bytes_peak": 96101990400, + "reserved_bytes_peak_delta": 56857985024, + "segment_count_current": 511, + "segment_count_current_delta": 440, + "segment_count_peak": 511, + "segment_count_peak_delta": 440, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T132816253492Z.pickle", + "timestamp_utc": "2026-08-01T13:28:16.373683Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T132816253492Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T132816253492Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6182a5002937fc1b10116b2bd5a8d3a9c521aa7b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T132816253492Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e933dee0f6d8e076c3882a0b2417600ff1c8f8091b2221a9ddd513817e5f72c +size 5950556 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T132817404496Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T132817404496Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4be81b5e57f7fb2cd1a6f9945ede5ef99589c109 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T132817404496Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103559986176, + "active_bytes_peak_delta": 23636751872, + "after": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103559986176, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2507049472, + "inactive_split_bytes_peak": 4972350976, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:17.404086Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214459392, + "allocated_bytes_peak_delta": 23643546624, + "before": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 79923234304, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 79570912768, + "device": 7, + "inactive_split_bytes_current": 2450426368, + "inactive_split_bytes_peak": 4972350976, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 96101990400, + "reserved_bytes_peak": 96101990400, + "segment_count_current": 511, + "segment_count_peak": 511, + "timestamp_utc": "2026-08-01T13:28:16.393959Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2507049472, + "inactive_split_bytes_current_delta": 56623104, + "inactive_split_bytes_peak": 4972350976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 10261364736, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 10261364736, + "segment_count_current": 580, + "segment_count_current_delta": 69, + "segment_count_peak": 580, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T132817404496Z.pickle", + "timestamp_utc": "2026-08-01T13:28:17.526417Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T132817404496Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T132817404496Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e4d86806176fda18d8cba5bf1c4c916fdbda6278 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T132817404496Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8b16c6226a5ebd08489a8e9c734fec07d609adc317e7a13c2b8d46b60b248a5 +size 5977988 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T132818527155Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T132818527155Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f85d1128905f54283b67fc0d6845b5e1d7bca76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T132818527155Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564865024, + "active_bytes_peak_delta": 4878848, + "after": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2746124800, + "inactive_split_bytes_peak": 4972350976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:18.526662Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214459392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103559986176, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2507049472, + "inactive_split_bytes_peak": 4972350976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:17.541239Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2746124800, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 4972350976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T132818527155Z.pickle", + "timestamp_utc": "2026-08-01T13:28:18.635221Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T132818527155Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T132818527155Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c917a5945b2fa9c2bbbb40a5b3fe665456638eaa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T132818527155Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61993490510677478d029bff66db89c62ad2b6acd9f2f4e2c746754116027b22 +size 5974918 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T132819923848Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T132819923848Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cde7c27b97cd254c5ff3fe1e1be18a4507a4a246 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T132819923848Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564865024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_peak": 4972350976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:19.923415Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214459392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2746124800, + "inactive_split_bytes_peak": 4972350976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:18.649089Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_current_delta": 117440512, + "inactive_split_bytes_peak": 4972350976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T132819923848Z.pickle", + "timestamp_utc": "2026-08-01T13:28:20.033976Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T132819923848Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T132819923848Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5752fff57f3b39e25e1a5c414ccdf1035829524 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T132819923848Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a838b341b6da292c69fe75e5ea813dc69fbb6cf2b10cabe3b9c4605e733d472b +size 5974335 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T132821308012Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T132821308012Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c620fc5633138ec02e15a69e290fd7e81455d529 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T132821308012Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564865024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_peak": 4972350976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:21.307578Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214459392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_peak": 4972350976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:20.048354Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4972350976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T132821308012Z.pickle", + "timestamp_utc": "2026-08-01T13:28:21.417740Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T132821308012Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T132821308012Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f871f9224ea86b45cf4ad37a5b096acd96b322e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T132821308012Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f3d6d906116ef5b332b71eaa164e486c3aa2aff6577eae2a8f9c9e0da359f37 +size 5974365 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T132822447999Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T132822447999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..999e77e857f6c46c3e185aea238689dfd82a80ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T132822447999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564865024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 3041823232, + "inactive_split_bytes_peak": 4972350976, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:22.447585Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214459392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_peak": 4972350976, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:21.431837Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3041823232, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 4972350976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T132822447999Z.pickle", + "timestamp_utc": "2026-08-01T13:28:22.552737Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T132822447999Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T132822447999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ac60c545e66746d4419130a15a0c6ffd5750375 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T132822447999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d713cfed32636409e56dfd9bc4484e9c7129279c16b99584934d28eedaae48c3 +size 5974400 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T132823575847Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T132823575847Z.json new file mode 100644 index 0000000000000000000000000000000000000000..266b83b49c095ce413ffec397f23071c3a68b147 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T132823575847Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564865024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_peak": 4972350976, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:23.575453Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214459392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 3041823232, + "inactive_split_bytes_peak": 4972350976, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:22.566352Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 4972350976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T132823575847Z.pickle", + "timestamp_utc": "2026-08-01T13:28:23.687040Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T132823575847Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T132823575847Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..474823f675690117ae2165832a367de8af1c9d12 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T132823575847Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb72507d273ec5226c22c2244ba2c10dfcaee65d1f04bb63f6fe5f6f56589ac3 +size 5974335 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T132824879827Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T132824879827Z.json new file mode 100644 index 0000000000000000000000000000000000000000..84e96b57868c026542b2116a480fb30c65d7a517 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T132824879827Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564865024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_peak": 4972350976, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:24.879432Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214459392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_peak": 4972350976, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:23.703097Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4972350976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T132824879827Z.pickle", + "timestamp_utc": "2026-08-01T13:28:24.985692Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T132824879827Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T132824879827Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e0ff3fc3135b0a6c1182a25c23ed3c4bf98613a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T132824879827Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12bdd74f6d2fd91dce06b893c97b15c1eb72eac71b4b87229b848247063bd585 +size 5974366 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T132826001273Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T132826001273Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9e55c5fbebc63b177b15e65aed6675ec1cfa7189 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T132826001273Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440968704, + "active_bytes_current_delta": 0, + "active_bytes_peak": 103564865024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_peak": 4972350976, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:26.000874Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440968704, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 103214459392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_peak": 4972350976, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:24.999329Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 4972350976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T132826001273Z.pickle", + "timestamp_utc": "2026-08-01T13:28:26.106637Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T132826001273Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T132826001273Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44b135f7d6fcc5ab536c10acaaeb31cc74333cd8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T132826001273Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b9130a8838fca0191c812376be09bf546ea650145e3f5e805f4b00dde5687bc +size 5974366 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T132921090507Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T132921090507Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a072f92af03716db7ccbf893aac471a99cd9710c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T132921090507Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166679552, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 103564865024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166679552, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 61166679552, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2976811520, + "inactive_split_bytes_peak": 4972350976, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:29:21.090031Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166679552, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 103214459392, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440968704, + "active_bytes_peak": 103564865024, + "allocated_bytes_current": 72440968704, + "allocated_bytes_peak": 103214459392, + "device": 7, + "inactive_split_bytes_current": 2863565312, + "inactive_split_bytes_peak": 4972350976, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106363355136, + "reserved_bytes_peak": 106363355136, + "segment_count_current": 580, + "segment_count_peak": 580, + "timestamp_utc": "2026-08-01T13:28:26.120635Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 2976811520, + "inactive_split_bytes_current_delta": 113246208, + "inactive_split_bytes_peak": 4972350976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106363355136, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106363355136, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 580, + "segment_count_current_delta": 0, + "segment_count_peak": 580, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T132921090507Z.pickle", + "timestamp_utc": "2026-08-01T13:29:21.295833Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T132921090507Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T132921090507Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7d46486bcbc935a8f0fb241a111b3a57a9afef94 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T132921090507Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7978e725de9cc5ab10599dea2af091d23e3d0228763f5919fd4479262588c425 +size 9738988 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..fccba7bbcf4cdb261f46340ffefa5bb2c84229fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 2048 +capacity: 1.25 +expert_dispatch_overlap: off +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6028 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 220 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T133205847250Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T133205847250Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c16a633a4d534849223b25649d118058784a91f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T133205847250Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 31622601728, + "active_bytes_peak": 80915032064, + "active_bytes_peak_delta": 45448695808, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 80915032064, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 80915032064, + "device": 0, + "inactive_split_bytes_current": 5726276608, + "inactive_split_bytes_peak": 7096372224, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 95747571712, + "reserved_bytes_peak": 95747571712, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:32:05.846824Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 31622601728, + "allocated_bytes_peak": 80915032064, + "allocated_bytes_peak_delta": 45448695808, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:30:58.083580Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5726276608, + "inactive_split_bytes_current_delta": 5591360512, + "inactive_split_bytes_peak": 7096372224, + "inactive_split_bytes_peak_delta": 6935740416, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 95747571712, + "reserved_bytes_current_delta": 56501469184, + "reserved_bytes_peak": 95747571712, + "reserved_bytes_peak_delta": 56501469184, + "segment_count_current": 505, + "segment_count_current_delta": 433, + "segment_count_peak": 505, + "segment_count_peak_delta": 433, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T133205847250Z.pickle", + "timestamp_utc": "2026-08-01T13:32:05.958141Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T133205847250Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T133205847250Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6c2c070f5a8fde13167187d79d7ef7c7c8b2aa4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T133205847250Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40d03913862ef138fe5014f10165e2f75013df1414467954c27af33c8ddfbeae +size 5381472 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T133206846174Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T133206846174Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d7df2173b18ff025f88d518c4b0c90156e6d787f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T133206846174Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559437824, + "active_bytes_peak_delta": 23644405760, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104559437824, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104559437824, + "device": 0, + "inactive_split_bytes_current": 5564795904, + "inactive_split_bytes_peak": 7486893056, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:06.845754Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559437824, + "allocated_bytes_peak_delta": 23644405760, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 80915032064, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 80915032064, + "device": 0, + "inactive_split_bytes_current": 5726276608, + "inactive_split_bytes_peak": 7096372224, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 95747571712, + "reserved_bytes_peak": 95747571712, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:32:05.975560Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5564795904, + "inactive_split_bytes_current_delta": -161480704, + "inactive_split_bytes_peak": 7486893056, + "inactive_split_bytes_peak_delta": 390520832, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_current_delta": 10609491968, + "reserved_bytes_peak": 106357063680, + "reserved_bytes_peak_delta": 10609491968, + "segment_count_current": 565, + "segment_count_current_delta": 60, + "segment_count_peak": 565, + "segment_count_peak_delta": 60, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T133206846174Z.pickle", + "timestamp_utc": "2026-08-01T13:32:06.959902Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T133206846174Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T133206846174Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d49aee158f22933caef79c18c40cab3b9c4d1d50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T133206846174Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1193b9b3e65bae0f7734ff1deb7f543e5cf94c4796e801c4b1000017d18641c9 +size 5607897 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T133207823045Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T133207823045Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96deee2891e14f115a41358141a832b60752a6a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T133207823045Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104564637696, + "active_bytes_peak_delta": 5199872, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 5826939904, + "inactive_split_bytes_peak": 7609282560, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:07.822658Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104564637696, + "allocated_bytes_peak_delta": 5199872, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104559437824, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104559437824, + "device": 0, + "inactive_split_bytes_current": 5564795904, + "inactive_split_bytes_peak": 7486893056, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:06.973203Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5826939904, + "inactive_split_bytes_current_delta": 262144000, + "inactive_split_bytes_peak": 7609282560, + "inactive_split_bytes_peak_delta": 122389504, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106357063680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 565, + "segment_count_current_delta": 0, + "segment_count_peak": 565, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T133207823045Z.pickle", + "timestamp_utc": "2026-08-01T13:32:07.923491Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T133207823045Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T133207823045Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..73f50f97e47ddbdc1c494af4bb806388e7b546a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T133207823045Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9dc488b6f400f83751986c9bfa76ace313bd0d914a4dfcc7bc317f8659dd4a8 +size 5607159 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T133208808292Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T133208808292Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a2571500b638bcaba62c61605b26043b778a084 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T133208808292Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104564637696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 5499784192, + "inactive_split_bytes_peak": 7609282560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:08.807886Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104564637696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 5826939904, + "inactive_split_bytes_peak": 7609282560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:07.936340Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5499784192, + "inactive_split_bytes_current_delta": -327155712, + "inactive_split_bytes_peak": 7609282560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106357063680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 565, + "segment_count_current_delta": 0, + "segment_count_peak": 565, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T133208808292Z.pickle", + "timestamp_utc": "2026-08-01T13:32:08.913023Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T133208808292Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T133208808292Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ccd45e187a8ceadc8a55651973c1341dbe57303 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T133208808292Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fea0f5acf8433898cc5dc464b6a3ac855e61341b7191a66f8639267e4c3d446 +size 5607722 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T133210260480Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T133210260480Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a70fc6dc8797199fc6420316650e5799d73ad9b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T133210260480Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104564637696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 5824842752, + "inactive_split_bytes_peak": 7621857280, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:10.260051Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104564637696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 5499784192, + "inactive_split_bytes_peak": 7609282560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:08.925863Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5824842752, + "inactive_split_bytes_current_delta": 325058560, + "inactive_split_bytes_peak": 7621857280, + "inactive_split_bytes_peak_delta": 12574720, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106357063680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 565, + "segment_count_current_delta": 0, + "segment_count_peak": 565, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T133210260480Z.pickle", + "timestamp_utc": "2026-08-01T13:32:10.364848Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T133210260480Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T133210260480Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e493ef83075555f480d4cf217986c814844b028 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T133210260480Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bab0488a69dbe5208059075f090e4db3a5abd2d7efd1bb2f1c71172ac02761f8 +size 5606203 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T133211248627Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T133211248627Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7bb11f75876e695f904a5362d92eef78bc994154 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T133211248627Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104564637696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 5707402240, + "inactive_split_bytes_peak": 7621857280, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:11.248224Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104564637696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 5824842752, + "inactive_split_bytes_peak": 7621857280, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:10.377873Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5707402240, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 7621857280, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106357063680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 565, + "segment_count_current_delta": 0, + "segment_count_peak": 565, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T133211248627Z.pickle", + "timestamp_utc": "2026-08-01T13:32:11.349482Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T133211248627Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T133211248627Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e180ada51dadcd67beea00223967f1fbfcd99f52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T133211248627Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:088e80279fbce361bd78b1d2c43ee57f0d0b60fad7ef00d37e05acfc5303bc61 +size 5607594 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T133212231729Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T133212231729Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7bffadc57cee55ace0d92052bba6b4b9021df94c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T133212231729Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104564637696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 5854202880, + "inactive_split_bytes_peak": 7621857280, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:12.231261Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104564637696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 5707402240, + "inactive_split_bytes_peak": 7621857280, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:11.362117Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5854202880, + "inactive_split_bytes_current_delta": 146800640, + "inactive_split_bytes_peak": 7621857280, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106357063680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 565, + "segment_count_current_delta": 0, + "segment_count_peak": 565, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T133212231729Z.pickle", + "timestamp_utc": "2026-08-01T13:32:12.329346Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T133212231729Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T133212231729Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2f062687375e78a626c5cfd24afafcaa7089c4e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T133212231729Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b7b15d2e34148ff9ac92c21d03528925203e7ded68e1460db9da2bf3e4d4f49 +size 5606068 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T133213206587Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T133213206587Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0a1cccbb515d5afcb000949f0705f5c08f4e524e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T133213206587Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104564637696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 6036655104, + "inactive_split_bytes_peak": 7621857280, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:13.206190Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104564637696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 5854202880, + "inactive_split_bytes_peak": 7621857280, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:12.341778Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6036655104, + "inactive_split_bytes_current_delta": 182452224, + "inactive_split_bytes_peak": 7621857280, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106357063680, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 565, + "segment_count_current_delta": 0, + "segment_count_peak": 565, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T133213206587Z.pickle", + "timestamp_utc": "2026-08-01T13:32:13.302978Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T133213206587Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T133213206587Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0bec52017887ba9875e869906073c91e552125c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T133213206587Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e10c746076b2da98aa0e38f63516d6ae885c090f62fe7930e704b6e13d4330b +size 5607295 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T133214179972Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T133214179972Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd07188f3a7da52b17e95f9231391815c7bf0737 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T133214179972Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104564637696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 5826939904, + "inactive_split_bytes_peak": 7621857280, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106359160832, + "reserved_bytes_peak": 106359160832, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T13:32:14.179532Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104564637696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 6036655104, + "inactive_split_bytes_peak": 7621857280, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106357063680, + "reserved_bytes_peak": 106357063680, + "segment_count_current": 565, + "segment_count_peak": 565, + "timestamp_utc": "2026-08-01T13:32:13.315339Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5826939904, + "inactive_split_bytes_current_delta": -209715200, + "inactive_split_bytes_peak": 7621857280, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106359160832, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 106359160832, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 566, + "segment_count_current_delta": 1, + "segment_count_peak": 566, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T133214179972Z.pickle", + "timestamp_utc": "2026-08-01T13:32:14.276979Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T133214179972Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T133214179972Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2799a3fd8208f68d18d89b3ac2f9b0bd7fbf0b0d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T133214179972Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22d15366074bfb36d9b4d2d5698e04280e24b4b4b1f31624e4438445c5ffd781 +size 5606011 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T133307992389Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T133307992389Z.json new file mode 100644 index 0000000000000000000000000000000000000000..29c1bc5f1dd83b2233385eeaf57c9f9065971a6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T133307992389Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803414016, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104564637696, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803414016, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66803414016, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 4313107456, + "inactive_split_bytes_peak": 7621857280, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106359160832, + "reserved_bytes_peak": 106359160832, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T13:33:07.991903Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803414016, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104564637696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 104564637696, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 104564637696, + "device": 0, + "inactive_split_bytes_current": 5826939904, + "inactive_split_bytes_peak": 7621857280, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106359160832, + "reserved_bytes_peak": 106359160832, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T13:32:14.289340Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4313107456, + "inactive_split_bytes_current_delta": -1513832448, + "inactive_split_bytes_peak": 7621857280, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106359160832, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106359160832, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T133307992389Z.pickle", + "timestamp_utc": "2026-08-01T13:33:08.189084Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T133307992389Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T133307992389Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e90eeb22533ff261a7d30989b34d36644060d66c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T133307992389Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21e4ac6fdfe438b7c1acb4897039e93508295ff62c3d8c96736f8f1d57d09255 +size 9626940 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T133205847237Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T133205847237Z.json new file mode 100644 index 0000000000000000000000000000000000000000..880f54527f317eba0b4d077cd883a07a8ec6ac7c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T133205847237Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 80919513088, + "active_bytes_peak_delta": 45453176832, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80919513088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80919513088, + "device": 1, + "inactive_split_bytes_current": 5789699072, + "inactive_split_bytes_peak": 7000930304, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 95745474560, + "reserved_bytes_peak": 95745474560, + "segment_count_current": 504, + "segment_count_peak": 504, + "timestamp_utc": "2026-08-01T13:32:05.846677Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 80919513088, + "allocated_bytes_peak_delta": 45453176832, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:30:58.084010Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5789699072, + "inactive_split_bytes_current_delta": 5656880128, + "inactive_split_bytes_peak": 7000930304, + "inactive_split_bytes_peak_delta": 6840544256, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 95745474560, + "reserved_bytes_current_delta": 56501469184, + "reserved_bytes_peak": 95745474560, + "reserved_bytes_peak_delta": 56501469184, + "segment_count_current": 504, + "segment_count_current_delta": 433, + "segment_count_peak": 504, + "segment_count_peak_delta": 433, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T133205847237Z.pickle", + "timestamp_utc": "2026-08-01T13:32:05.955522Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T133205847237Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T133205847237Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d9c0290621d24264511231c81bb073b5e12da49e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T133205847237Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e8d18050e7c7dbc15e66ae2078b622ff6a2eb630b2eb528cd8f28be2181024b +size 5372894 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T133206846184Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T133206846184Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f5bad038dcc57e320c7937198d9b7ed565fdfdc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T133206846184Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 23642103808, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5609344000, + "inactive_split_bytes_peak": 7346062336, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:06.845762Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 23642103808, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80919513088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80919513088, + "device": 1, + "inactive_split_bytes_current": 5789699072, + "inactive_split_bytes_peak": 7000930304, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 95745474560, + "reserved_bytes_peak": 95745474560, + "segment_count_current": 504, + "segment_count_peak": 504, + "timestamp_utc": "2026-08-01T13:32:05.973200Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5609344000, + "inactive_split_bytes_current_delta": -180355072, + "inactive_split_bytes_peak": 7346062336, + "inactive_split_bytes_peak_delta": 345132032, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 10546577408, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 10546577408, + "segment_count_current": 561, + "segment_count_current_delta": 57, + "segment_count_peak": 561, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T133206846184Z.pickle", + "timestamp_utc": "2026-08-01T13:32:06.954584Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T133206846184Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T133206846184Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bab8e937b175dc6e23dc127638c9153bae6f92e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T133206846184Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:414a06b300030aa1e1c0fd16b3052c2231c01d43bfeaed4f7acb1df171121165 +size 5568693 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T133207823084Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T133207823084Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f1b93cd095efc0791aca8e96e2727b5893d8d77 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T133207823084Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5575789568, + "inactive_split_bytes_peak": 7757515776, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:07.822695Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5609344000, + "inactive_split_bytes_peak": 7346062336, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:06.967838Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5575789568, + "inactive_split_bytes_current_delta": -33554432, + "inactive_split_bytes_peak": 7757515776, + "inactive_split_bytes_peak_delta": 411453440, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T133207823084Z.pickle", + "timestamp_utc": "2026-08-01T13:32:07.923658Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T133207823084Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T133207823084Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4eb286695d41b2afb2642006f705a72e018f9c80 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T133207823084Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4b5c9af72c62ca3ea69a7e33dc4a706168728311c094d8143a9099626e0a199 +size 5570219 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T133208808323Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T133208808323Z.json new file mode 100644 index 0000000000000000000000000000000000000000..38337206e0243c2d425c28d3e1a0b4332a24ebba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T133208808323Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5785504768, + "inactive_split_bytes_peak": 7757515776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:08.807920Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5575789568, + "inactive_split_bytes_peak": 7757515776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:07.936390Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5785504768, + "inactive_split_bytes_current_delta": 209715200, + "inactive_split_bytes_peak": 7757515776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T133208808323Z.pickle", + "timestamp_utc": "2026-08-01T13:32:08.908127Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T133208808323Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T133208808323Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb8987ab23a05d1e78cf5e50267810d0a043316e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T133208808323Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24edd6c3d2ca3dd51d84ad7847316cdd00467b2190c615241ee1c0c9d725228b +size 5568420 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T133210260492Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T133210260492Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d3c95811d0718009619c34d535ff654459b798aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T133210260492Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5873585152, + "inactive_split_bytes_peak": 7757515776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:10.260085Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5785504768, + "inactive_split_bytes_peak": 7757515776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:08.924382Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5873585152, + "inactive_split_bytes_current_delta": 88080384, + "inactive_split_bytes_peak": 7757515776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T133210260492Z.pickle", + "timestamp_utc": "2026-08-01T13:32:10.368853Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T133210260492Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T133210260492Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aab157178b2e331ca088eaaeb62c675c6aa8d4d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T133210260492Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:917f8926b8a4b005ecfbfd117ec3a5fc2dc5592201a45bf5a2a00ad643b9cc45 +size 5569778 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T133211248672Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T133211248672Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5c96e51b1c78ae306c0535e96079fcd52154ac84 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T133211248672Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5875682304, + "inactive_split_bytes_peak": 7757515776, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:11.248270Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5873585152, + "inactive_split_bytes_peak": 7757515776, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:10.384303Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5875682304, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 7757515776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T133211248672Z.pickle", + "timestamp_utc": "2026-08-01T13:32:11.347529Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T133211248672Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T133211248672Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..45cf98581913dde7b75c880642837571abc9c17b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T133211248672Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a5573659b3b7ab29c20138d31cdc02151ff03a57e7de035e3f4a44a98bfe636 +size 5599951 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T133212231712Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T133212231712Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a4bf3fcd87c06979de76b10591a33bc669ec297 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T133212231712Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5758241792, + "inactive_split_bytes_peak": 7757515776, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:12.231309Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5875682304, + "inactive_split_bytes_peak": 7757515776, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:11.360052Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5758241792, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 7757515776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T133212231712Z.pickle", + "timestamp_utc": "2026-08-01T13:32:12.330424Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T133212231712Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T133212231712Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..83f339273fa551a18e9edd10ce0693fcd37e0d22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T133212231712Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ddd99c6159ea2fb606708b340c9c5d860fadcb914f55bfb7a7a5447ef35ecc7 +size 5569155 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T133213206612Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T133213206612Z.json new file mode 100644 index 0000000000000000000000000000000000000000..20da8bb0c554375e6b58beca916f4ca46c4a444d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T133213206612Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5586275328, + "inactive_split_bytes_peak": 7782087680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:13.206210Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5758241792, + "inactive_split_bytes_peak": 7757515776, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:12.342885Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5586275328, + "inactive_split_bytes_current_delta": -171966464, + "inactive_split_bytes_peak": 7782087680, + "inactive_split_bytes_peak_delta": 24571904, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T133213206612Z.pickle", + "timestamp_utc": "2026-08-01T13:32:13.303563Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T133213206612Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T133213206612Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29672b63398cde33e18599b548663c0faa2e955f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T133213206612Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2811cfcb66c1c558d2c1cd03f4d61b0b71f4fafda15022b966ea7fca54486f9 +size 5568924 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T133214179973Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T133214179973Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94e1fc2770c965a057d903bff881d3d19b6ff118 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T133214179973Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5728881664, + "inactive_split_bytes_peak": 7782087680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:14.179558Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5586275328, + "inactive_split_bytes_peak": 7782087680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:13.315953Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5728881664, + "inactive_split_bytes_current_delta": 142606336, + "inactive_split_bytes_peak": 7782087680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T133214179973Z.pickle", + "timestamp_utc": "2026-08-01T13:32:14.277988Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T133214179973Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T133214179973Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f225653a306755493c27bd078bc12dfd19d3103d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T133214179973Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d72cd7fcfd1a3c73d747c22497a33377e64ad4f80eebb6c1af1b23b273cc44e +size 5568491 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T133307992455Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T133307992455Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6b29b1a421b2061d8b9ff4387c40d012340729e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T133307992455Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 4334586880, + "inactive_split_bytes_peak": 7782087680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:33:07.991895Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 1, + "inactive_split_bytes_current": 5728881664, + "inactive_split_bytes_peak": 7782087680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:14.290527Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4334586880, + "inactive_split_bytes_current_delta": -1394294784, + "inactive_split_bytes_peak": 7782087680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T133307992455Z.pickle", + "timestamp_utc": "2026-08-01T13:33:08.188025Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T133307992455Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T133307992455Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..271ce1976b5eede2c70e726a2aa87c83c6345e66 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T133307992455Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac956be1a81c50a6141f8040616b1a47da14fc50478ac06d05c69052a0c10460 +size 9170321 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T133205847206Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T133205847206Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06c058949f65fd8abd75ea2f403d13ed1a2bc6cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T133205847206Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 80919513088, + "active_bytes_peak_delta": 45453176832, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80919513088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80919513088, + "device": 2, + "inactive_split_bytes_current": 5787601920, + "inactive_split_bytes_peak": 6883416064, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 95745474560, + "reserved_bytes_peak": 95745474560, + "segment_count_current": 504, + "segment_count_peak": 504, + "timestamp_utc": "2026-08-01T13:32:05.846638Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 80919513088, + "allocated_bytes_peak_delta": 45453176832, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:30:58.084093Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5787601920, + "inactive_split_bytes_current_delta": 5654782976, + "inactive_split_bytes_peak": 6883416064, + "inactive_split_bytes_peak_delta": 6723030016, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 95745474560, + "reserved_bytes_current_delta": 56501469184, + "reserved_bytes_peak": 95745474560, + "reserved_bytes_peak_delta": 56501469184, + "segment_count_current": 504, + "segment_count_current_delta": 433, + "segment_count_peak": 504, + "segment_count_peak_delta": 433, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T133205847206Z.pickle", + "timestamp_utc": "2026-08-01T13:32:05.955525Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T133205847206Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T133205847206Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6de0c385789e11eb94e406bec8acb0b9aec6948b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T133205847206Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:203b6769be5719810f6104dc44debf3865e10adea441ae45daec5f1ed84611c0 +size 5372913 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T133206846172Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T133206846172Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b6694f10d6dff20e7ba610141a7e8b2bca039c34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T133206846172Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 23642103808, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5607246848, + "inactive_split_bytes_peak": 7346062336, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:06.845763Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 23642103808, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80919513088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80919513088, + "device": 2, + "inactive_split_bytes_current": 5787601920, + "inactive_split_bytes_peak": 6883416064, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 95745474560, + "reserved_bytes_peak": 95745474560, + "segment_count_current": 504, + "segment_count_peak": 504, + "timestamp_utc": "2026-08-01T13:32:05.972835Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5607246848, + "inactive_split_bytes_current_delta": -180355072, + "inactive_split_bytes_peak": 7346062336, + "inactive_split_bytes_peak_delta": 462646272, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 10546577408, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 10546577408, + "segment_count_current": 561, + "segment_count_current_delta": 57, + "segment_count_peak": 561, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T133206846172Z.pickle", + "timestamp_utc": "2026-08-01T13:32:06.955955Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T133206846172Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T133206846172Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c9d01cbe3c040f936226aae768420a0625f10bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T133206846172Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b54a56853e67ef91efc34ab48c96cd0e2f8abffcc1567b9d41621eabb56def89 +size 5568624 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T133207823089Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T133207823089Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e89ac6ae7466d064ae4073de2d4d06ae3c4ca7e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T133207823089Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5582081024, + "inactive_split_bytes_peak": 7757515776, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:07.822689Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5607246848, + "inactive_split_bytes_peak": 7346062336, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:06.969305Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5582081024, + "inactive_split_bytes_current_delta": -25165824, + "inactive_split_bytes_peak": 7757515776, + "inactive_split_bytes_peak_delta": 411453440, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T133207823089Z.pickle", + "timestamp_utc": "2026-08-01T13:32:07.922753Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T133207823089Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T133207823089Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2f5b450407e548e5846d059daa39e2b3ceb22fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T133207823089Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48129728471019a961274ebaeb0cabedcb0b8bc755b06db0ec2dd0e39ddc63e5 +size 5570303 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T133208808291Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T133208808291Z.json new file mode 100644 index 0000000000000000000000000000000000000000..593c0df2103dce3a6208be3b05b4867fc96357cf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T133208808291Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5787601920, + "inactive_split_bytes_peak": 7757515776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:08.807890Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5582081024, + "inactive_split_bytes_peak": 7757515776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:07.935709Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5787601920, + "inactive_split_bytes_current_delta": 205520896, + "inactive_split_bytes_peak": 7757515776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T133208808291Z.pickle", + "timestamp_utc": "2026-08-01T13:32:08.909347Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T133208808291Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T133208808291Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7b8851015f79c8a571fe3ae6a747dfc1830167a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T133208808291Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1706009ca30decca6153a15e65c1f47857499a888575e9ee0b7f976d48be407c +size 5602458 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T133210260466Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T133210260466Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a49496178d078d2da4a5461558ce2be9b4fc855e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T133210260466Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5875682304, + "inactive_split_bytes_peak": 7757515776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:10.260046Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5787601920, + "inactive_split_bytes_peak": 7757515776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:08.922125Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5875682304, + "inactive_split_bytes_current_delta": 88080384, + "inactive_split_bytes_peak": 7757515776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T133210260466Z.pickle", + "timestamp_utc": "2026-08-01T13:32:10.363132Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T133210260466Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T133210260466Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf9a1f7ee93c5d87cc1043670fc8b276f16063bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T133210260466Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68072215f2a46a831002fdae455e5600acb5bdc2949a0af78e8ed7573b15b90c +size 5569990 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T133211248629Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T133211248629Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5369b4510294d0c94abc414ac54bfcd891ffa8f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T133211248629Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5758241792, + "inactive_split_bytes_peak": 7757515776, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:11.248223Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5875682304, + "inactive_split_bytes_peak": 7757515776, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:10.376268Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5758241792, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 7757515776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T133211248629Z.pickle", + "timestamp_utc": "2026-08-01T13:32:11.349191Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T133211248629Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T133211248629Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b5383533b186c31efc754cf3f07b3c89cb568676 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T133211248629Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb60ded7af1a558f9c70fc49d7a935449e94edccbc77f7cd96d2036ab22b5a8c +size 5568374 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T133212231700Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T133212231700Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1835117a891837a85c612d00dcb36bd8e012068b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T133212231700Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5584178176, + "inactive_split_bytes_peak": 7782812672, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:12.231277Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5758241792, + "inactive_split_bytes_peak": 7757515776, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:11.361925Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5584178176, + "inactive_split_bytes_current_delta": -174063616, + "inactive_split_bytes_peak": 7782812672, + "inactive_split_bytes_peak_delta": 25296896, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T133212231700Z.pickle", + "timestamp_utc": "2026-08-01T13:32:12.328944Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T133212231700Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T133212231700Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..90f876b919efd2fc9d7874c2f0225edef73953d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T133212231700Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c11e1135109598a9cedd294fcb9f87eac33e41c9e15dc42631d6de3a9c61df6 +size 5602350 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T133213206588Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T133213206588Z.json new file mode 100644 index 0000000000000000000000000000000000000000..07e825e611f9ed45db88387ee3fc0b6d7701777c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T133213206588Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5611441152, + "inactive_split_bytes_peak": 7782812672, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:13.206184Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5584178176, + "inactive_split_bytes_peak": 7782812672, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:12.341500Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5611441152, + "inactive_split_bytes_current_delta": 27262976, + "inactive_split_bytes_peak": 7782812672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T133213206588Z.pickle", + "timestamp_utc": "2026-08-01T13:32:13.303265Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T133213206588Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T133213206588Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b51ad0c2e7a1786c25b8b6ac4b9f2f54783d8205 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T133213206588Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a109f6e74d29679fb9b0ba911f15a111ff744314a8bff41857d171591c2271e9 +size 5583987 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T133214179921Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T133214179921Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31123c14902d66c8b73608b4df0baf263fee7dd7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T133214179921Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5877779456, + "inactive_split_bytes_peak": 7782812672, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:14.179525Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5611441152, + "inactive_split_bytes_peak": 7782812672, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:13.315632Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5877779456, + "inactive_split_bytes_current_delta": 266338304, + "inactive_split_bytes_peak": 7782812672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T133214179921Z.pickle", + "timestamp_utc": "2026-08-01T13:32:14.275640Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T133214179921Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T133214179921Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dc47d2506ea679862d960d9b7500374b551a6eff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T133214179921Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4ace29ec29ee089a07ee4d12071b88bcdbc6a5e12aea36148b989389544ff02 +size 5568598 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T133307992451Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T133307992451Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2190dbc981fc76ef71701ca51cd399843b8d9b73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T133307992451Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104561616896, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 4334586880, + "inactive_split_bytes_peak": 7782812672, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:33:07.991911Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104561616896, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104561616896, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104561616896, + "device": 2, + "inactive_split_bytes_current": 5877779456, + "inactive_split_bytes_peak": 7782812672, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:14.287973Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4334586880, + "inactive_split_bytes_current_delta": -1543192576, + "inactive_split_bytes_peak": 7782812672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T133307992451Z.pickle", + "timestamp_utc": "2026-08-01T13:33:08.188273Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T133307992451Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T133307992451Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a289dc6fa7aa8947cbf384fa77d975aa69b78960 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T133307992451Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14a3379f7076487e13e403fbecc2acbf2a822c2d7d29bf536bba095e33809f97 +size 9170354 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T133205847345Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T133205847345Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f134967b75728f0e70ed6fd81ba8c6d7ac168b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T133205847345Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 80918464512, + "active_bytes_peak_delta": 45452128256, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80918464512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80918464512, + "device": 3, + "inactive_split_bytes_current": 5720493056, + "inactive_split_bytes_peak": 6976296960, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 95745474560, + "reserved_bytes_peak": 95745474560, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T13:32:05.846777Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 80918464512, + "allocated_bytes_peak_delta": 45452128256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:30:58.084277Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5720493056, + "inactive_split_bytes_current_delta": 5587674112, + "inactive_split_bytes_peak": 6976296960, + "inactive_split_bytes_peak_delta": 6815910912, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 95745474560, + "reserved_bytes_current_delta": 56501469184, + "reserved_bytes_peak": 95745474560, + "reserved_bytes_peak_delta": 56501469184, + "segment_count_current": 502, + "segment_count_current_delta": 431, + "segment_count_peak": 502, + "segment_count_peak_delta": 431, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T133205847345Z.pickle", + "timestamp_utc": "2026-08-01T13:32:05.956280Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T133205847345Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T133205847345Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7a934c21b79a246ebe18afe3370c0101b0a43c08 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T133205847345Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05f7e586ee792574fa625529002a4bfa55636ea69630900bd713e8a3fe0de841 +size 5372570 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T133206846167Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T133206846167Z.json new file mode 100644 index 0000000000000000000000000000000000000000..88937dc048ff4c234661686ed60a16613be777d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T133206846167Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104562203648, + "active_bytes_peak_delta": 23643739136, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104562203648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104562203648, + "device": 3, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_peak": 7524808192, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:06.845751Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104562203648, + "allocated_bytes_peak_delta": 23643739136, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80918464512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80918464512, + "device": 3, + "inactive_split_bytes_current": 5720493056, + "inactive_split_bytes_peak": 6976296960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 95745474560, + "reserved_bytes_peak": 95745474560, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T13:32:05.974580Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_current_delta": -165675008, + "inactive_split_bytes_peak": 7524808192, + "inactive_split_bytes_peak_delta": 548511232, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 10546577408, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 10546577408, + "segment_count_current": 561, + "segment_count_current_delta": 59, + "segment_count_peak": 561, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T133206846167Z.pickle", + "timestamp_utc": "2026-08-01T13:32:06.956417Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T133206846167Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T133206846167Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..603f7b6874ffe0e849b8fcbbb72c207ba572c2eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T133206846167Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e346b48ba93ee5f874668864693fd2b78250931d3cbab1be78f95e49fe604789 +size 5569619 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T133207823060Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T133207823060Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d82a3c372cc32764ef4bf764f959f7b60765186b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T133207823060Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104562203648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104562203648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104562203648, + "device": 3, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_peak": 7524808192, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:07.822673Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104562203648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104562203648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104562203648, + "device": 3, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_peak": 7524808192, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:06.969716Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_current_delta": 257949696, + "inactive_split_bytes_peak": 7524808192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T133207823060Z.pickle", + "timestamp_utc": "2026-08-01T13:32:07.923768Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T133207823060Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T133207823060Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3559ce9fc24de6c9cb191613101f825af6035275 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T133207823060Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8b0919c48889693908ed366f48781ab96b87c3e14c244c55d95ba1b4e071196 +size 5588806 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T133208808291Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T133208808291Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dc24dcc5815f42ad42339fbf4dc3df1b1dc00d7e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T133208808291Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 1757696, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 5814864896, + "inactive_split_bytes_peak": 7524808192, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:08.807890Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 1757696, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104562203648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104562203648, + "device": 3, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_peak": 7524808192, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:07.936632Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5814864896, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 7524808192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T133208808291Z.pickle", + "timestamp_utc": "2026-08-01T13:32:08.910091Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T133208808291Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T133208808291Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f2d67549ed7efd304a1d314f113e3eddfd1378dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T133208808291Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12800ff21d2acecc065c78b5868970781057a7eac5a86a2b73b59855ba0b7d33 +size 5579065 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T133210260467Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T133210260467Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a54c59b584c1a386b51533f581c1382b7db6c011 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T133210260467Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 5808573440, + "inactive_split_bytes_peak": 7524808192, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:10.260049Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 5814864896, + "inactive_split_bytes_peak": 7524808192, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:08.922793Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5808573440, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 7524808192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T133210260467Z.pickle", + "timestamp_utc": "2026-08-01T13:32:10.365319Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T133210260467Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T133210260467Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..84bdb9df231e3588d4b298d2b1e417059e93f5b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T133210260467Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3985f0f72db00146ba30b100cce880e786d888864f9762d0367f3175a589f72c +size 5569649 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T133211248615Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T133211248615Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cea5971a9eec997e515fbf90410f9194ba7ca84a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T133211248615Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 5810670592, + "inactive_split_bytes_peak": 7524808192, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:11.248220Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 5808573440, + "inactive_split_bytes_peak": 7524808192, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:10.378355Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5810670592, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 7524808192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T133211248615Z.pickle", + "timestamp_utc": "2026-08-01T13:32:11.347730Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T133211248615Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T133211248615Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b63f9310bd0d377098ed7d9f789cb4c2fccd193d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T133211248615Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ad4f1b015d6a51fe19ddb1424340266d8314bd4fb3a76e7f03b55909d89262b +size 5584766 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T133212231693Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T133212231693Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d6696b788d557c99686373a450c41f6b2c7124a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T133212231693Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 5816962048, + "inactive_split_bytes_peak": 7524808192, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:12.231295Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 5810670592, + "inactive_split_bytes_peak": 7524808192, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:11.360389Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5816962048, + "inactive_split_bytes_current_delta": 6291456, + "inactive_split_bytes_peak": 7524808192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T133212231693Z.pickle", + "timestamp_utc": "2026-08-01T13:32:12.336065Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T133212231693Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T133212231693Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..81e920defcf96445e43f9f5e5278139f081ce0ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T133212231693Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2ecf5ca4fd101396da72294011243900feb6446feacd410631a6dcf351e0036 +size 5569931 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T133213206590Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T133213206590Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cde901beebfc744c801b7fbdc96f59e3f1332622 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T133213206590Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 5816962048, + "inactive_split_bytes_peak": 7524808192, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:13.206194Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 5816962048, + "inactive_split_bytes_peak": 7524808192, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:12.350777Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5816962048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7524808192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T133213206590Z.pickle", + "timestamp_utc": "2026-08-01T13:32:13.303499Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T133213206590Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T133213206590Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8527e22157b3e1fca03ca057656ddcab0357217c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T133213206590Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b902378eeb31f750f7af66cde05e31467f6b1c6c21dee9458bcaff87d6b0cfb0 +size 5603107 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T133214179987Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T133214179987Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94f2e98501e79ebeb0d46d000c8c1b01025c2f9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T133214179987Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 5814864896, + "inactive_split_bytes_peak": 7524808192, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:14.179526Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 5816962048, + "inactive_split_bytes_peak": 7524808192, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:13.315739Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5814864896, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7524808192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T133214179987Z.pickle", + "timestamp_utc": "2026-08-01T13:32:14.276452Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T133214179987Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T133214179987Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..26be380c334382d3d99a937d31b34fcf84985aa1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T133214179987Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:557f74de7f92912b0f128fe58a51d9d1fcc9fa2cb5f73e94bdee4cbecaaea5f7 +size 5569768 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T133307992404Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T133307992404Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e74445c58a1e21da40d8f01f83e8a11f2f9c560 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T133307992404Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 4452027392, + "inactive_split_bytes_peak": 7524808192, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:33:07.991905Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 3, + "inactive_split_bytes_current": 5814864896, + "inactive_split_bytes_peak": 7524808192, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:14.288713Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4452027392, + "inactive_split_bytes_current_delta": -1362837504, + "inactive_split_bytes_peak": 7524808192, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T133307992404Z.pickle", + "timestamp_utc": "2026-08-01T13:33:08.188465Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T133307992404Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T133307992404Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ccb556c3485b0af8e6077c949e94fecff4814c1a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T133307992404Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d2e03481262463004f7c84d20be14527bc3a4f2f1585aa646d28d6f7fe625f9 +size 9171626 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T133205847230Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T133205847230Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ceb1cb5ff9cb2e452ad851526c7ad05f06fd145 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T133205847230Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855589888, + "active_bytes_current_delta": 31623692288, + "active_bytes_peak": 80924682240, + "active_bytes_peak_delta": 45457903616, + "after": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 80924682240, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 80924682240, + "device": 4, + "inactive_split_bytes_current": 5150124032, + "inactive_split_bytes_peak": 6638871552, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 95745474560, + "reserved_bytes_peak": 95745474560, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T13:32:05.846820Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855589888, + "allocated_bytes_current_delta": 31623692288, + "allocated_bytes_peak": 80924682240, + "allocated_bytes_peak_delta": 45457903616, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:30:58.083628Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5150124032, + "inactive_split_bytes_current_delta": 5017747456, + "inactive_split_bytes_peak": 6638871552, + "inactive_split_bytes_peak_delta": 6485462016, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 95745474560, + "reserved_bytes_current_delta": 56501469184, + "reserved_bytes_peak": 95745474560, + "reserved_bytes_peak_delta": 56501469184, + "segment_count_current": 502, + "segment_count_current_delta": 431, + "segment_count_peak": 502, + "segment_count_peak_delta": 431, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T133205847230Z.pickle", + "timestamp_utc": "2026-08-01T13:32:05.958677Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T133205847230Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T133205847230Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..851b7f74eaf8cca4197d7547b8198150842b70c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T133205847230Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bb308e1806f40caca44ef7084cb360db14bb30c1f8d00bcb3e8960eceecd497 +size 5380776 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T133206846152Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T133206846152Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae29104fe13de89d71a38f0fbb7bc5959dfaa0dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T133206846152Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855589888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558209536, + "active_bytes_peak_delta": 23633527296, + "after": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104558209536, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104558209536, + "device": 4, + "inactive_split_bytes_current": 5160609792, + "inactive_split_bytes_peak": 7116838912, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:06.845750Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855589888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558209536, + "allocated_bytes_peak_delta": 23633527296, + "before": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 80924682240, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 80924682240, + "device": 4, + "inactive_split_bytes_current": 5150124032, + "inactive_split_bytes_peak": 6638871552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 95745474560, + "reserved_bytes_peak": 95745474560, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T13:32:05.976320Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5160609792, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 7116838912, + "inactive_split_bytes_peak_delta": 477967360, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106159931392, + "reserved_bytes_current_delta": 10414456832, + "reserved_bytes_peak": 106159931392, + "reserved_bytes_peak_delta": 10414456832, + "segment_count_current": 561, + "segment_count_current_delta": 59, + "segment_count_peak": 561, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T133206846152Z.pickle", + "timestamp_utc": "2026-08-01T13:32:06.955505Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T133206846152Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T133206846152Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1dfbe348cea367bd36258c0776c6ec1c07a14c1c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T133206846152Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6d452fe688e82e36937323ad0280be29e180d22646ab12fd3803530f660f5da +size 5574429 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T133207823069Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T133207823069Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4304c0a9c295b8f097cb586f7832e0dd4e811193 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T133207823069Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855589888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558209536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104558209536, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104558209536, + "device": 4, + "inactive_split_bytes_current": 5630371840, + "inactive_split_bytes_peak": 7116838912, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:07.822678Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855589888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558209536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104558209536, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104558209536, + "device": 4, + "inactive_split_bytes_current": 5160609792, + "inactive_split_bytes_peak": 7116838912, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:06.968708Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5630371840, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 7116838912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106159931392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106159931392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T133207823069Z.pickle", + "timestamp_utc": "2026-08-01T13:32:07.923500Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T133207823069Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T133207823069Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1329be1af4b961c6a64d187701d994413b894484 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T133207823069Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1af654ef1ebfc18fa2d09962c7e6f8228afb2f1e2be298fba974d4e78003c16 +size 5594374 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T133208808295Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T133208808295Z.json new file mode 100644 index 0000000000000000000000000000000000000000..889b15c83d2355b3e24f29653f744d9d737b1c02 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T133208808295Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855589888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558209536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104558209536, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104558209536, + "device": 4, + "inactive_split_bytes_current": 5483571200, + "inactive_split_bytes_peak": 7116838912, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:08.807891Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855589888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558209536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104558209536, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104558209536, + "device": 4, + "inactive_split_bytes_current": 5630371840, + "inactive_split_bytes_peak": 7116838912, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:07.936116Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5483571200, + "inactive_split_bytes_current_delta": -146800640, + "inactive_split_bytes_peak": 7116838912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106159931392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106159931392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T133208808295Z.pickle", + "timestamp_utc": "2026-08-01T13:32:08.908619Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T133208808295Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T133208808295Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..23447d1c76dc3f21839b597d9d2802b4977df88a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T133208808295Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7718e61191572fd89d0e84386235f1858a9110abe43e8c0fd411204cb647ae0a +size 5578317 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T133210260469Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T133210260469Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d6dcd8d8349b8d70584b47000a13a93f94153780 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T133210260469Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855589888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558209536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104558209536, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104558209536, + "device": 4, + "inactive_split_bytes_current": 5515028480, + "inactive_split_bytes_peak": 7116838912, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:10.260048Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855589888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558209536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104558209536, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104558209536, + "device": 4, + "inactive_split_bytes_current": 5483571200, + "inactive_split_bytes_peak": 7116838912, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:08.921634Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5515028480, + "inactive_split_bytes_current_delta": 31457280, + "inactive_split_bytes_peak": 7116838912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106159931392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106159931392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T133210260469Z.pickle", + "timestamp_utc": "2026-08-01T13:32:10.364826Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T133210260469Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T133210260469Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9b06006f5db58fdf26446e73ba12507e74d28b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T133210260469Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7ce14a69a98e4fb428f78f8cb6891ab768b4e3171bd629dfcecf3a565e686d6 +size 5577583 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T133211248644Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T133211248644Z.json new file mode 100644 index 0000000000000000000000000000000000000000..08c8cad8326af28d8350795112e6cbd4ed0c39a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T133211248644Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855589888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558209536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104558209536, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104558209536, + "device": 4, + "inactive_split_bytes_current": 5928167424, + "inactive_split_bytes_peak": 7320922112, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:11.248237Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855589888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558209536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104558209536, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104558209536, + "device": 4, + "inactive_split_bytes_current": 5515028480, + "inactive_split_bytes_peak": 7116838912, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:10.379644Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5928167424, + "inactive_split_bytes_current_delta": 413138944, + "inactive_split_bytes_peak": 7320922112, + "inactive_split_bytes_peak_delta": 204083200, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106159931392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106159931392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T133211248644Z.pickle", + "timestamp_utc": "2026-08-01T13:32:11.353929Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T133211248644Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T133211248644Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ccbc988aa8bc20ad4749adbb8db547c3e657333a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T133211248644Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29a6e89ff92acd95b6ae0b3b3821d7b6b1fbdfb01d0d67ae09cba7136b1d0e20 +size 5577368 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T133212231681Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T133212231681Z.json new file mode 100644 index 0000000000000000000000000000000000000000..935cc7a6a2555926b3de57c364783cd283b2b4f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T133212231681Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855589888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558209536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104558209536, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104558209536, + "device": 4, + "inactive_split_bytes_current": 5571651584, + "inactive_split_bytes_peak": 7320922112, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:12.231274Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855589888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558209536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104558209536, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104558209536, + "device": 4, + "inactive_split_bytes_current": 5928167424, + "inactive_split_bytes_peak": 7320922112, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:11.368866Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5571651584, + "inactive_split_bytes_current_delta": -356515840, + "inactive_split_bytes_peak": 7320922112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106159931392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106159931392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T133212231681Z.pickle", + "timestamp_utc": "2026-08-01T13:32:12.329569Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T133212231681Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T133212231681Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b65ca86810bc357869dfda827a4a27ec4deae96c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T133212231681Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59b4bcd6e6d6bca2ead1995993c0b62c5b976c2598156f5564e93bb68f2cced8 +size 5577085 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T133213206589Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T133213206589Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e9cbaa241f2acf0de4f6c4d174be48e632175586 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T133213206589Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855589888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561748992, + "active_bytes_peak_delta": 3539456, + "after": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104561748992, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104561748992, + "device": 4, + "inactive_split_bytes_current": 5571651584, + "inactive_split_bytes_peak": 7320922112, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:13.206190Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855589888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561748992, + "allocated_bytes_peak_delta": 3539456, + "before": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104558209536, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104558209536, + "device": 4, + "inactive_split_bytes_current": 5571651584, + "inactive_split_bytes_peak": 7320922112, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:12.342018Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5571651584, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7320922112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106159931392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106159931392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T133213206589Z.pickle", + "timestamp_utc": "2026-08-01T13:32:13.303037Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T133213206589Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T133213206589Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ccf85b24c2b09c286706eb4e01ffd32a0f89707 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T133213206589Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97a4e4bfd905637b1bc6818b72966c049b3c39ec39983fe26e15f084e3ad8851 +size 5574626 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T133214179942Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T133214179942Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea586eb34030b2644d3ff5b05ecd56f66b12c051 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T133214179942Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855589888, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104561748992, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104561748992, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104561748992, + "device": 4, + "inactive_split_bytes_current": 5454211072, + "inactive_split_bytes_peak": 7320922112, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106162028544, + "reserved_bytes_peak": 106162028544, + "segment_count_current": 562, + "segment_count_peak": 562, + "timestamp_utc": "2026-08-01T13:32:14.179537Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855589888, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104561748992, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104561748992, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104561748992, + "device": 4, + "inactive_split_bytes_current": 5571651584, + "inactive_split_bytes_peak": 7320922112, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106159931392, + "reserved_bytes_peak": 106159931392, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:13.315368Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5454211072, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 7320922112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106162028544, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 106162028544, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 562, + "segment_count_current_delta": 1, + "segment_count_peak": 562, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T133214179942Z.pickle", + "timestamp_utc": "2026-08-01T13:32:14.278270Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T133214179942Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T133214179942Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9916520231160335c8f492ff0d668fe31e39b3af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T133214179942Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:553fc85d7b4cd7460f3b9a852ad1578c7d88d5734648a9005dffc37942362fe7 +size 5578804 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T133307992475Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T133307992475Z.json new file mode 100644 index 0000000000000000000000000000000000000000..da54785cbcc652ae614fb80393353b461a14b0e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T133307992475Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804946944, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104561748992, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66804946944, + "active_bytes_peak": 104561748992, + "allocated_bytes_current": 66804946944, + "allocated_bytes_peak": 104561748992, + "device": 4, + "inactive_split_bytes_current": 4431112192, + "inactive_split_bytes_peak": 7320922112, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106162028544, + "reserved_bytes_peak": 106162028544, + "segment_count_current": 562, + "segment_count_peak": 562, + "timestamp_utc": "2026-08-01T13:33:07.991901Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804946944, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104561748992, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855589888, + "active_bytes_peak": 104561748992, + "allocated_bytes_current": 66855589888, + "allocated_bytes_peak": 104561748992, + "device": 4, + "inactive_split_bytes_current": 5454211072, + "inactive_split_bytes_peak": 7320922112, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106162028544, + "reserved_bytes_peak": 106162028544, + "segment_count_current": 562, + "segment_count_peak": 562, + "timestamp_utc": "2026-08-01T13:32:14.291314Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4431112192, + "inactive_split_bytes_current_delta": -1023098880, + "inactive_split_bytes_peak": 7320922112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106162028544, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106162028544, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 562, + "segment_count_current_delta": 0, + "segment_count_peak": 562, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T133307992475Z.pickle", + "timestamp_utc": "2026-08-01T13:33:08.195284Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T133307992475Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T133307992475Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de243a80b61a9eab61799bc3c80e59639f00271d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T133307992475Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61d69027e844d35ca95a802558cb1a2b2dfb11e4c41885f11002b592b551936f +size 9260656 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T133205847241Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T133205847241Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf802bf7793def51c3b50e426a1176efb616967c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T133205847241Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 80927209472, + "active_bytes_peak_delta": 45460873216, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80927209472, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80927209472, + "device": 5, + "inactive_split_bytes_current": 5588372480, + "inactive_split_bytes_peak": 7320906752, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 96202653696, + "reserved_bytes_peak": 96202653696, + "segment_count_current": 506, + "segment_count_peak": 506, + "timestamp_utc": "2026-08-01T13:32:05.846658Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 80927209472, + "allocated_bytes_peak_delta": 45460873216, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:30:58.084082Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5588372480, + "inactive_split_bytes_current_delta": 5455553536, + "inactive_split_bytes_peak": 7320906752, + "inactive_split_bytes_peak_delta": 7160520704, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 96202653696, + "reserved_bytes_current_delta": 56958648320, + "reserved_bytes_peak": 96202653696, + "reserved_bytes_peak_delta": 56958648320, + "segment_count_current": 506, + "segment_count_current_delta": 435, + "segment_count_peak": 506, + "segment_count_peak_delta": 435, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T133205847241Z.pickle", + "timestamp_utc": "2026-08-01T13:32:05.956594Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T133205847241Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T133205847241Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e96f4e9c55dba3ebbb2b6e45f6768c75b2ec662 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T133205847241Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80121d42b10cdb127e689e2247e9aa3984692c1aa43e75f773e048924e52ef7e +size 5373152 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T133206846170Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T133206846170Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c80b4e0707a9c5aca908819fce30279d4ee1692 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T133206846170Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558960128, + "active_bytes_peak_delta": 23631750656, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558960128, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558960128, + "device": 5, + "inactive_split_bytes_current": 5653384192, + "inactive_split_bytes_peak": 7344241664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:06.845750Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558960128, + "allocated_bytes_peak_delta": 23631750656, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80927209472, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80927209472, + "device": 5, + "inactive_split_bytes_current": 5588372480, + "inactive_split_bytes_peak": 7320906752, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 96202653696, + "reserved_bytes_peak": 96202653696, + "segment_count_current": 506, + "segment_count_peak": 506, + "timestamp_utc": "2026-08-01T13:32:05.974133Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5653384192, + "inactive_split_bytes_current_delta": 65011712, + "inactive_split_bytes_peak": 7344241664, + "inactive_split_bytes_peak_delta": 23334912, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 10076815360, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 10076815360, + "segment_count_current": 560, + "segment_count_current_delta": 54, + "segment_count_peak": 560, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T133206846170Z.pickle", + "timestamp_utc": "2026-08-01T13:32:06.954978Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T133206846170Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T133206846170Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d87a68ffcf084e2d9ba2eb000b781e3a47cf70a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T133206846170Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d8056f66f5e389aa7238b6b4a0a4313ec47e57c881a37920d0d3412e707d5a7 +size 5567682 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T133207823053Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T133207823053Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7179e2057fde2dba14a5cc713038af7f3d173aa3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T133207823053Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566597632, + "active_bytes_peak_delta": 7637504, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 5917625344, + "inactive_split_bytes_peak": 7519324160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:07.822666Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566597632, + "allocated_bytes_peak_delta": 7637504, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558960128, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558960128, + "device": 5, + "inactive_split_bytes_current": 5653384192, + "inactive_split_bytes_peak": 7344241664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:06.968220Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5917625344, + "inactive_split_bytes_current_delta": 264241152, + "inactive_split_bytes_peak": 7519324160, + "inactive_split_bytes_peak_delta": 175082496, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T133207823053Z.pickle", + "timestamp_utc": "2026-08-01T13:32:07.923294Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T133207823053Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T133207823053Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8194ae50af92346fd9534f97426b9bdb3207051f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T133207823053Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:506e00cfab4887b19ad42474d8bbd69b8591915f78e15f27508e46b0e3188dc7 +size 5569166 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T133208808293Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T133208808293Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a68444ab698acd2ace3aa1c85b666c002d39d63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T133208808293Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566597632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 5596761088, + "inactive_split_bytes_peak": 7519324160, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:08.807874Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566597632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 5917625344, + "inactive_split_bytes_peak": 7519324160, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:07.936042Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5596761088, + "inactive_split_bytes_current_delta": -320864256, + "inactive_split_bytes_peak": 7519324160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T133208808293Z.pickle", + "timestamp_utc": "2026-08-01T13:32:08.909107Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T133208808293Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T133208808293Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d95f6bfc81f03755767249c81ba2d2d313d5580 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T133208808293Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f47e8bf8e5d627cad8e2ebf454969237fab0a96d4768114769052e1304d2893 +size 5565797 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T133210260447Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T133210260447Z.json new file mode 100644 index 0000000000000000000000000000000000000000..621948a8b9da8bd386b468a93932d74f4b71efd6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T133210260447Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566597632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 5919722496, + "inactive_split_bytes_peak": 7519324160, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:10.260029Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566597632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 5596761088, + "inactive_split_bytes_peak": 7519324160, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:08.921743Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5919722496, + "inactive_split_bytes_current_delta": 322961408, + "inactive_split_bytes_peak": 7519324160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T133210260447Z.pickle", + "timestamp_utc": "2026-08-01T13:32:10.363285Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T133210260447Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T133210260447Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d87f5487521b845b761bb53971acf2ccf0ee949d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T133210260447Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41f9a590811507bd83a8083a6c5c517e4253dd0c8f601fee1a8f5d44d3bae867 +size 5568384 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T133211248668Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T133211248668Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cdc071e5a89525868d25f6c8db26ec6bfd80241c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T133211248668Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566597632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 5919722496, + "inactive_split_bytes_peak": 7519324160, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:11.248250Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566597632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 5919722496, + "inactive_split_bytes_peak": 7519324160, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:10.376423Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5919722496, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7519324160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T133211248668Z.pickle", + "timestamp_utc": "2026-08-01T13:32:11.346887Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T133211248668Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T133211248668Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c13448ba6ea133b5313b13e978932ae8e3edb087 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T133211248668Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0437853782db568c8431e1ed3eb1b7c7af1d3b4473ff3f25cc8f9dfa8055723 +size 5568082 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T133212231715Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T133212231715Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3d1801670c277862ed173b3648e083b18e720560 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T133212231715Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566597632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 5923916800, + "inactive_split_bytes_peak": 7519324160, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:12.231292Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566597632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 5919722496, + "inactive_split_bytes_peak": 7519324160, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:11.359610Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5923916800, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 7519324160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T133212231715Z.pickle", + "timestamp_utc": "2026-08-01T13:32:12.329917Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T133212231715Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T133212231715Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1209dc885d8a8fb7faf105944fc26cd47ef51ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T133212231715Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f90b1caa61237fb8d0a4e8c913694cfa5ad03b3bc00c88d0da44a5cf8e6d76a7 +size 5568033 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T133213206589Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T133213206589Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1b358d6bbf41a8af1c8087b5ae8b79a3f2dd3f03 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T133213206589Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566597632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 6009900032, + "inactive_split_bytes_peak": 7519324160, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:13.206184Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566597632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 5923916800, + "inactive_split_bytes_peak": 7519324160, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:12.344280Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6009900032, + "inactive_split_bytes_current_delta": 85983232, + "inactive_split_bytes_peak": 7519324160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T133213206589Z.pickle", + "timestamp_utc": "2026-08-01T13:32:13.306993Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T133213206589Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T133213206589Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca95211fe23bafcf9f70bfdf8f5b18abcd324986 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T133213206589Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16c50d71aa4799872c4c14718b5a006cc45142c0d26aa33bea126f2c4bb8ebb0 +size 5566992 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T133214179929Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T133214179929Z.json new file mode 100644 index 0000000000000000000000000000000000000000..be32965cb9e2febc9daa0e6c151967e79990350c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T133214179929Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566597632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 5921819648, + "inactive_split_bytes_peak": 7519324160, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:14.179525Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566597632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 6009900032, + "inactive_split_bytes_peak": 7519324160, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:13.322735Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5921819648, + "inactive_split_bytes_current_delta": -88080384, + "inactive_split_bytes_peak": 7519324160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T133214179929Z.pickle", + "timestamp_utc": "2026-08-01T13:32:14.280211Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T133214179929Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T133214179929Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..40c9e9d8c9499fd72777292ddb46fa1d905696c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T133214179929Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d39cf243caff656b0a7ee36502a5344b6eddc29a7e398436c2e2b6b66a98e0c1 +size 5568043 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T133307992386Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T133307992386Z.json new file mode 100644 index 0000000000000000000000000000000000000000..487031420f7050b811a77fb07a5414e5a732bc5d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T133307992386Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104566597632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 4498164736, + "inactive_split_bytes_peak": 7519324160, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:33:07.991898Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104566597632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104566597632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104566597632, + "device": 5, + "inactive_split_bytes_current": 5921819648, + "inactive_split_bytes_peak": 7519324160, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:14.292791Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4498164736, + "inactive_split_bytes_current_delta": -1423654912, + "inactive_split_bytes_peak": 7519324160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T133307992386Z.pickle", + "timestamp_utc": "2026-08-01T13:33:08.193347Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T133307992386Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T133307992386Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4cb8ba80f314e783d04a85c0e1eebcc4dbc2ce71 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T133307992386Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6be2b08fa38ed4af974b369eef5db9f4f876325e135363e52134e3fee4ab84a3 +size 9169759 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T133205847257Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T133205847257Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ec236245a23066f39e6f75d27e0a2d81ed48b65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T133205847257Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 80918464512, + "active_bytes_peak_delta": 45452128256, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80918464512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80918464512, + "device": 6, + "inactive_split_bytes_current": 5726784512, + "inactive_split_bytes_peak": 6867245056, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 95745474560, + "reserved_bytes_peak": 95745474560, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T13:32:05.846691Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 80918464512, + "allocated_bytes_peak_delta": 45452128256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:30:58.084177Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5726784512, + "inactive_split_bytes_current_delta": 5593965568, + "inactive_split_bytes_peak": 6867245056, + "inactive_split_bytes_peak_delta": 6706859008, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 95745474560, + "reserved_bytes_current_delta": 56501469184, + "reserved_bytes_peak": 95745474560, + "reserved_bytes_peak_delta": 56501469184, + "segment_count_current": 502, + "segment_count_current_delta": 431, + "segment_count_peak": 502, + "segment_count_peak_delta": 431, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T133205847257Z.pickle", + "timestamp_utc": "2026-08-01T13:32:05.954931Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T133205847257Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T133205847257Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f4c249d656a7e6b6c9c4a134a4f47a8eacf57ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T133205847257Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8db4691ecfe4d1fbd3134407beae8fa003d73b579c1e8d7f7a24042cf416ec99 +size 5372591 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T133206846187Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T133206846187Z.json new file mode 100644 index 0000000000000000000000000000000000000000..21e15c652a3454f5e5da1db0d5ab0c2aba0407e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T133206846187Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104562203648, + "active_bytes_peak_delta": 23643739136, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104562203648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104562203648, + "device": 6, + "inactive_split_bytes_current": 5552720896, + "inactive_split_bytes_peak": 7520613888, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:06.845781Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104562203648, + "allocated_bytes_peak_delta": 23643739136, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80918464512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80918464512, + "device": 6, + "inactive_split_bytes_current": 5726784512, + "inactive_split_bytes_peak": 6867245056, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 95745474560, + "reserved_bytes_peak": 95745474560, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T13:32:05.972137Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5552720896, + "inactive_split_bytes_current_delta": -174063616, + "inactive_split_bytes_peak": 7520613888, + "inactive_split_bytes_peak_delta": 653368832, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 10546577408, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 10546577408, + "segment_count_current": 561, + "segment_count_current_delta": 59, + "segment_count_peak": 561, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T133206846187Z.pickle", + "timestamp_utc": "2026-08-01T13:32:06.959258Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T133206846187Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T133206846187Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..88e4b1083b201b3673a4107c7587b7f5a76f88e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T133206846187Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9422671df24dd9edacd655634a5c27a1ffb85853f71144716e7a75196a73954d +size 5578287 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T133207823097Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T133207823097Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fbdde56e3d6622a1b19e3dea694f8fee123cd9ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T133207823097Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104562203648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104562203648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104562203648, + "device": 6, + "inactive_split_bytes_current": 5814864896, + "inactive_split_bytes_peak": 7520613888, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:07.822696Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104562203648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104562203648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104562203648, + "device": 6, + "inactive_split_bytes_current": 5552720896, + "inactive_split_bytes_peak": 7520613888, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:06.972975Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5814864896, + "inactive_split_bytes_current_delta": 262144000, + "inactive_split_bytes_peak": 7520613888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T133207823097Z.pickle", + "timestamp_utc": "2026-08-01T13:32:07.925195Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T133207823097Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T133207823097Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cd8d0c57fd6102318b9ea00337b83993c0266fbd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T133207823097Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c04a340df8b32fd3a1fae102b7bb31756122e7a142ca1d9a9a553c9f0c84a087 +size 5569503 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T133208808319Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T133208808319Z.json new file mode 100644 index 0000000000000000000000000000000000000000..716658a2bb9324af22fa175d8fec112ff6bc92b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T133208808319Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 1757696, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_peak": 7520613888, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:08.807920Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 1757696, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104562203648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104562203648, + "device": 6, + "inactive_split_bytes_current": 5814864896, + "inactive_split_bytes_peak": 7520613888, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:07.938292Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7520613888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T133208808319Z.pickle", + "timestamp_utc": "2026-08-01T13:32:08.914718Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T133208808319Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T133208808319Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2063e73d8a3181c6489756530051168ae3bb7508 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T133208808319Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf6c5ff11469713fee6f418db925b08d6080b95598d5762be791ab18d41aee1e +size 5570058 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T133210260475Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T133210260475Z.json new file mode 100644 index 0000000000000000000000000000000000000000..092bae586abf845f7d6c4e2d9a567cc5b5b6646b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T133210260475Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 5814864896, + "inactive_split_bytes_peak": 7520613888, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:10.260061Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_peak": 7520613888, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:08.930202Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5814864896, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 7520613888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T133210260475Z.pickle", + "timestamp_utc": "2026-08-01T13:32:10.365793Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T133210260475Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T133210260475Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..18a2a304e5f96568a76e705cd2ba8ddf9e6029dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T133210260475Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e4ba6afe506d603260f697d21d118caec53a1283c24009059d8b2209b364bf7 +size 5566054 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T133211248644Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T133211248644Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b59260b9c965f0b4c6bac89da4232ebcbe5d98d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T133211248644Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_peak": 7520613888, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:11.248247Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 5814864896, + "inactive_split_bytes_peak": 7520613888, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:10.378800Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7520613888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T133211248644Z.pickle", + "timestamp_utc": "2026-08-01T13:32:11.349217Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T133211248644Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T133211248644Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a90c67c64329e7dbc9e098e654fc1840bf999eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T133211248644Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec537a628ea374f529231e78c301c24fa776755bc8e659c9238aec6d55fbc710 +size 5578675 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T133212231697Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T133212231697Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c9c676f6228b71e206aa69d199ee4fb565e727c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T133212231697Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 5816962048, + "inactive_split_bytes_peak": 7520613888, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:12.231288Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_peak": 7520613888, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:11.362007Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5816962048, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 7520613888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T133212231697Z.pickle", + "timestamp_utc": "2026-08-01T13:32:12.329977Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T133212231697Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T133212231697Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b00b1e278a6e77b3735a8d0ab64d2d48aec45d68 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T133212231697Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ef734caf8565e169458f9222ce270f212980e585a4f64b2119e9dc1a4146c47 +size 5569715 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T133213206617Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T133213206617Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ddc7b5d56ef698036f5ccce1b92839fd68f5e1a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T133213206617Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_peak": 7520613888, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:13.206200Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 5816962048, + "inactive_split_bytes_peak": 7520613888, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:12.342485Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 7520613888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T133213206617Z.pickle", + "timestamp_utc": "2026-08-01T13:32:13.306797Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T133213206617Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T133213206617Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0eaacefdcfe3999bb648ca704791e409702c27d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T133213206617Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ced7a132ec49eb3b3e364c4a75ecfb3914b3d6fbf599b9e5dd2a82bc028e060 +size 5569859 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T133214179944Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T133214179944Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0067c6da99eabaf23d0d342aff7b8bf5ba98ee98 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T133214179944Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_peak": 7520613888, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:14.179547Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_peak": 7520613888, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:13.319829Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7520613888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T133214179944Z.pickle", + "timestamp_utc": "2026-08-01T13:32:14.278532Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T133214179944Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T133214179944Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..19a47705706d60fbdf615206b39d9299f30e01d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T133214179944Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:732c59d2e2f8839de811a0ea680fb5c7a985a7ecb390ac611165d22f356b416a +size 5569358 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T133307992478Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T133307992478Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ff41424f4759f1361357d9eb9efe9d0df610e6dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T133307992478Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104563961344, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 4452027392, + "inactive_split_bytes_peak": 7520613888, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:33:07.991896Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104563961344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104563961344, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104563961344, + "device": 6, + "inactive_split_bytes_current": 5812767744, + "inactive_split_bytes_peak": 7520613888, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106292051968, + "reserved_bytes_peak": 106292051968, + "segment_count_current": 561, + "segment_count_peak": 561, + "timestamp_utc": "2026-08-01T13:32:14.291157Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4452027392, + "inactive_split_bytes_current_delta": -1360740352, + "inactive_split_bytes_peak": 7520613888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106292051968, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106292051968, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 561, + "segment_count_current_delta": 0, + "segment_count_peak": 561, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T133307992478Z.pickle", + "timestamp_utc": "2026-08-01T13:33:08.185565Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T133307992478Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T133307992478Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..66ea34bc93332c63d62c0bdbbe22ff7eab5ed1c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T133307992478Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:639644c47ebdc1853e8d54225b83b5abea2d36c7ffe206c8df7f356ba58abf06 +size 9171646 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T133205847128Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T133205847128Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5fc8bcc651fc24c3252f0aba8bb3fa612a8a6df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T133205847128Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 31622094336, + "active_bytes_peak": 80927209472, + "active_bytes_peak_delta": 45460873216, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 80927209472, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 80927209472, + "device": 7, + "inactive_split_bytes_current": 5588371968, + "inactive_split_bytes_peak": 7085861888, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 96202653696, + "reserved_bytes_peak": 96202653696, + "segment_count_current": 506, + "segment_count_peak": 506, + "timestamp_utc": "2026-08-01T13:32:05.846680Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 31622094336, + "allocated_bytes_peak": 80927209472, + "allocated_bytes_peak_delta": 45460873216, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:30:58.084469Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5588371968, + "inactive_split_bytes_current_delta": 5455553024, + "inactive_split_bytes_peak": 7085861888, + "inactive_split_bytes_peak_delta": 6925475840, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 96202653696, + "reserved_bytes_current_delta": 56958648320, + "reserved_bytes_peak": 96202653696, + "reserved_bytes_peak_delta": 56958648320, + "segment_count_current": 506, + "segment_count_current_delta": 435, + "segment_count_peak": 506, + "segment_count_peak_delta": 435, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T133205847128Z.pickle", + "timestamp_utc": "2026-08-01T13:32:05.954754Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T133205847128Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T133205847128Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1648afc556400c15a0e08674406d0d203f2d016 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T133205847128Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:943663cedd4c1fde88ba51df519234151c5bd4c5925bbd8f4f81877f17d60afb +size 5374747 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T133206846250Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T133206846250Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8cd3850e1d2420562a116e602993d9d09325bc5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T133206846250Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558960640, + "active_bytes_peak_delta": 23631751168, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104558960640, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104558960640, + "device": 7, + "inactive_split_bytes_current": 5657577984, + "inactive_split_bytes_peak": 7344241152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:06.845829Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558960640, + "allocated_bytes_peak_delta": 23631751168, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 80927209472, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 80927209472, + "device": 7, + "inactive_split_bytes_current": 5588371968, + "inactive_split_bytes_peak": 7085861888, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 96202653696, + "reserved_bytes_peak": 96202653696, + "segment_count_current": 506, + "segment_count_peak": 506, + "timestamp_utc": "2026-08-01T13:32:05.971992Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5657577984, + "inactive_split_bytes_current_delta": 69206016, + "inactive_split_bytes_peak": 7344241152, + "inactive_split_bytes_peak_delta": 258379264, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 10076815360, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 10076815360, + "segment_count_current": 560, + "segment_count_current_delta": 54, + "segment_count_peak": 560, + "segment_count_peak_delta": 54, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T133206846250Z.pickle", + "timestamp_utc": "2026-08-01T13:32:06.959699Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T133206846250Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T133206846250Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b0dd60938e82956d4110976e899a25cfed42854 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T133206846250Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:897e62cf0816bb1204b554246aada9aba7c5aca944bd30b9467d176b19146855 +size 5569582 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T133207823150Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T133207823150Z.json new file mode 100644 index 0000000000000000000000000000000000000000..732ba790089903c27359b1d9bfbbb80192535ac5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T133207823150Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566598144, + "active_bytes_peak_delta": 7637504, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5921819136, + "inactive_split_bytes_peak": 7523517952, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:07.822761Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566598144, + "allocated_bytes_peak_delta": 7637504, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104558960640, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104558960640, + "device": 7, + "inactive_split_bytes_current": 5657577984, + "inactive_split_bytes_peak": 7344241152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:06.973267Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5921819136, + "inactive_split_bytes_current_delta": 264241152, + "inactive_split_bytes_peak": 7523517952, + "inactive_split_bytes_peak_delta": 179276800, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T133207823150Z.pickle", + "timestamp_utc": "2026-08-01T13:32:07.926762Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T133207823150Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T133207823150Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..92d32c74c4a02453f97267f9bf6fac799504db32 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T133207823150Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6494cc90d908899d141a0c245908082149e429161dd23a86b7a88937c953f1d +size 5571601 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T133208808337Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T133208808337Z.json new file mode 100644 index 0000000000000000000000000000000000000000..acc8602aa3d27d0392424ab33fed02fcbb553b5b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T133208808337Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566598144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5596760576, + "inactive_split_bytes_peak": 7523517952, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:08.807933Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566598144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5921819136, + "inactive_split_bytes_peak": 7523517952, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:07.941904Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5596760576, + "inactive_split_bytes_current_delta": -325058560, + "inactive_split_bytes_peak": 7523517952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T133208808337Z.pickle", + "timestamp_utc": "2026-08-01T13:32:08.907695Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T133208808337Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T133208808337Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f7875aa41d6cd13dc49756fc229d69a975e2c4b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T133208808337Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21fac0514ebb6fcbf495622fc7c0b0111a0e583f43e1ad083c6db8b8de4a6ed1 +size 5603907 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T133210260511Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T133210260511Z.json new file mode 100644 index 0000000000000000000000000000000000000000..77a691c670349fc4769215d7718475db07c88f0c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T133210260511Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566598144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5806475776, + "inactive_split_bytes_peak": 7523517952, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:10.260088Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566598144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5596760576, + "inactive_split_bytes_peak": 7523517952, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:08.920803Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5806475776, + "inactive_split_bytes_current_delta": 209715200, + "inactive_split_bytes_peak": 7523517952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T133210260511Z.pickle", + "timestamp_utc": "2026-08-01T13:32:10.363737Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T133210260511Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T133210260511Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c5d25aa45ae2812a429cc3bed365f63c7d141e1b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T133210260511Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8b9531cb04513243ba62837bf55bdfd5aa92388414563432c2f2bda410e1611 +size 5570319 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T133211248674Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T133211248674Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea56d5ad6f261c43173c77db7a6cd5a8e0652fad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T133211248674Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566598144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5921819136, + "inactive_split_bytes_peak": 7523517952, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:11.248284Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566598144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5806475776, + "inactive_split_bytes_peak": 7523517952, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:10.377027Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5921819136, + "inactive_split_bytes_current_delta": 115343360, + "inactive_split_bytes_peak": 7523517952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T133211248674Z.pickle", + "timestamp_utc": "2026-08-01T13:32:11.347743Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T133211248674Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T133211248674Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..832bbc62f5060c5796a9fb8132e5a3fbd7e99d38 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T133211248674Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea84fe318b5852f2ae9b267bb93cafd2625807897565d9a84970a6dd396f4dc5 +size 5603393 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T133212231725Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T133212231725Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f001766d0595b98cfd5b18e75b31569dfecbe69 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T133212231725Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566598144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5923916288, + "inactive_split_bytes_peak": 7523517952, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:12.231327Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566598144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5921819136, + "inactive_split_bytes_peak": 7523517952, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:11.360476Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5923916288, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 7523517952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T133212231725Z.pickle", + "timestamp_utc": "2026-08-01T13:32:12.330199Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T133212231725Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T133212231725Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..990fdc84293cf7ce59289ef9ed374028911591c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T133212231725Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3deaef8b5ffe0e348c7627921b5e10b717d66132c97dc28b1f9bf456c8797269 +size 5585496 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T133213206629Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T133213206629Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c015b7130fa2b62448a4733e1d38c6a5de1ca16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T133213206629Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566598144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5919721984, + "inactive_split_bytes_peak": 7523517952, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:13.206240Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566598144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5923916288, + "inactive_split_bytes_peak": 7523517952, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:12.342718Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5919721984, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 7523517952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T133213206629Z.pickle", + "timestamp_utc": "2026-08-01T13:32:13.304222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T133213206629Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T133213206629Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..951d732e885a13896954797a2a6c85e6098b60f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T133213206629Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2a278266dedbb941e17d5caefe38c09130a74682d6bddd76a6200760cfb5a0e +size 5570258 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T133214180087Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T133214180087Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a15be10b2d90f8447eaf3b7f2f1c97cdfd7aa257 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T133214180087Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104566598144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5921819136, + "inactive_split_bytes_peak": 7523517952, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:14.179584Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104566598144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5919721984, + "inactive_split_bytes_peak": 7523517952, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:13.316639Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5921819136, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 7523517952, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T133214180087Z.pickle", + "timestamp_utc": "2026-08-01T13:32:14.278396Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T133214180087Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T133214180087Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f4a84af9b9c713e7d49658d35544061ae146a03f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T133214180087Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:673be577e61d06e7b704be0863bac552aaeca36ed21856d4e35e265f70fa86ff +size 5578955 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T133307992428Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T133307992428Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4058bd76d0d0d658992173c3dda43e82f110bc9c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T133307992428Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906624, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104566598144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906624, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66802906624, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 4498164224, + "inactive_split_bytes_peak": 7523935744, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:33:07.991901Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906624, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104566598144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104566598144, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104566598144, + "device": 7, + "inactive_split_bytes_current": 5921819136, + "inactive_split_bytes_peak": 7523517952, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106279469056, + "reserved_bytes_peak": 106279469056, + "segment_count_current": 560, + "segment_count_peak": 560, + "timestamp_utc": "2026-08-01T13:32:14.291307Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4498164224, + "inactive_split_bytes_current_delta": -1423654912, + "inactive_split_bytes_peak": 7523935744, + "inactive_split_bytes_peak_delta": 417792, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106279469056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106279469056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 560, + "segment_count_current_delta": 0, + "segment_count_peak": 560, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T133307992428Z.pickle", + "timestamp_utc": "2026-08-01T13:33:08.195199Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T133307992428Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T133307992428Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ab11922c9bc894be25b086f716259940a87226f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T133307992428Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f00fbc92f34bed3fb518a9df20f80d9a69b8bb6bb2167dcccf21404472a345ed +size 9214284 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..f680c6a52e93196a9246002bfdf074ab6b734503 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 2048 +capacity: 1.25 +expert_dispatch_overlap: off +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6029 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 220 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T133552447003Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T133552447003Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e912128f108fbce68e2b0989dc96f76f1b5d589 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T133552447003Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 37209103360, + "active_bytes_peak": 80833263616, + "active_bytes_peak_delta": 45366927360, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 80833263616, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 80833263616, + "device": 0, + "inactive_split_bytes_current": 6200544256, + "inactive_split_bytes_peak": 9074092544, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 92041904128, + "reserved_bytes_peak": 92041904128, + "segment_count_current": 424, + "segment_count_peak": 424, + "timestamp_utc": "2026-08-01T13:35:52.446439Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 37209103360, + "allocated_bytes_peak": 80833263616, + "allocated_bytes_peak_delta": 45366927360, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:34:43.477159Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6200544256, + "inactive_split_bytes_current_delta": 6065628160, + "inactive_split_bytes_peak": 9074092544, + "inactive_split_bytes_peak_delta": 8913460736, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 92041904128, + "reserved_bytes_current_delta": 52795801600, + "reserved_bytes_peak": 92041904128, + "reserved_bytes_peak_delta": 52795801600, + "segment_count_current": 424, + "segment_count_current_delta": 352, + "segment_count_peak": 424, + "segment_count_peak_delta": 352, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T133552447003Z.pickle", + "timestamp_utc": "2026-08-01T13:35:52.563425Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T133552447003Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T133552447003Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..54c1054c788b0f9310bd1098087b600dc13f0960 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T133552447003Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf49cbd7900e9fc4c71ad1c77ef14453d35158948beaff2ec5b862be261283a3 +size 5933668 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T133553571290Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T133553571290Z.json new file mode 100644 index 0000000000000000000000000000000000000000..acab73dd06d0a5ecad9265df436f83e153aee12a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T133553571290Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104469288448, + "active_bytes_peak_delta": 23636024832, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469288448, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469288448, + "device": 0, + "inactive_split_bytes_current": 6875827200, + "inactive_split_bytes_peak": 9074092544, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:35:53.570881Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469288448, + "allocated_bytes_peak_delta": 23636024832, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 80833263616, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 80833263616, + "device": 0, + "inactive_split_bytes_current": 6200544256, + "inactive_split_bytes_peak": 9074092544, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 92041904128, + "reserved_bytes_peak": 92041904128, + "segment_count_current": 424, + "segment_count_peak": 424, + "timestamp_utc": "2026-08-01T13:35:52.580317Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6875827200, + "inactive_split_bytes_current_delta": 675282944, + "inactive_split_bytes_peak": 9074092544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_current_delta": 14541651968, + "reserved_bytes_peak": 106583556096, + "reserved_bytes_peak_delta": 14541651968, + "segment_count_current": 490, + "segment_count_current_delta": 66, + "segment_count_peak": 490, + "segment_count_peak_delta": 66, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T133553571290Z.pickle", + "timestamp_utc": "2026-08-01T13:35:53.687382Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T133553571290Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T133553571290Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5db7f82a54de3f0cab3e9d5b4af43bacd9c6ab60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T133553571290Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e9c10b0b72595b176744e85c76f9bd1ee26a8d87f7a13114739441e387af2e2 +size 5948386 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T133554685263Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T133554685263Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a4d09a54a7a78bd1e9057126106c0bb18a6b4ecf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T133554685263Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104469288448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469288448, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469288448, + "device": 0, + "inactive_split_bytes_current": 6611586048, + "inactive_split_bytes_peak": 9074092544, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:35:54.684856Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469288448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469288448, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469288448, + "device": 0, + "inactive_split_bytes_current": 6875827200, + "inactive_split_bytes_peak": 9074092544, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:35:53.701441Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6611586048, + "inactive_split_bytes_current_delta": -264241152, + "inactive_split_bytes_peak": 9074092544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106583556096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T133554685263Z.pickle", + "timestamp_utc": "2026-08-01T13:35:54.789984Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T133554685263Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T133554685263Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d1a0003f4eee18e1b06240141ab508a4f8a34379 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T133554685263Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:161d83b12110d7f01ea6badb7f5a16088656e4e3341614d3006fa2bba4de5d9d +size 5942640 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T133556042404Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T133556042404Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5bc15f1b791d8e9f8e9ad25c18e049bd036d1f5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T133556042404Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104469288448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469288448, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469288448, + "device": 0, + "inactive_split_bytes_current": 6817106944, + "inactive_split_bytes_peak": 9074092544, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:35:56.041996Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469288448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469288448, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469288448, + "device": 0, + "inactive_split_bytes_current": 6611586048, + "inactive_split_bytes_peak": 9074092544, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:35:54.803637Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6817106944, + "inactive_split_bytes_current_delta": 205520896, + "inactive_split_bytes_peak": 9074092544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106583556096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T133556042404Z.pickle", + "timestamp_utc": "2026-08-01T13:35:56.149981Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T133556042404Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T133556042404Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60ef849e576b42546a245c700437da14464a7449 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T133556042404Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:281321293e81c3dabfc5671cc3d626a9907857393fa7aa7c9adf78afc234a0aa +size 6068041 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T133557413159Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T133557413159Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eb823fec6cd318b77238f78f96b14e1dd566d3c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T133557413159Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104469288448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469288448, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469288448, + "device": 0, + "inactive_split_bytes_current": 6611586048, + "inactive_split_bytes_peak": 9074092544, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:35:57.412729Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469288448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469288448, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469288448, + "device": 0, + "inactive_split_bytes_current": 6817106944, + "inactive_split_bytes_peak": 9074092544, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:35:56.163837Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6611586048, + "inactive_split_bytes_current_delta": -205520896, + "inactive_split_bytes_peak": 9074092544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106583556096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T133557413159Z.pickle", + "timestamp_utc": "2026-08-01T13:35:57.519091Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T133557413159Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T133557413159Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a3bd0442bd2e312493b59d269e40f3d4461cd79e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T133557413159Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3058f7f26270e83163a81954a43296900f8ca9582ba3458a0aff1ababa1fec25 +size 6067846 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T133558543082Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T133558543082Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7ad9fe207027e23e6619d774593c4ca950bc2299 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T133558543082Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104469897216, + "active_bytes_peak_delta": 608768, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469897216, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469897216, + "device": 0, + "inactive_split_bytes_current": 6905187328, + "inactive_split_bytes_peak": 9074092544, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:35:58.542684Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469897216, + "allocated_bytes_peak_delta": 608768, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469288448, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469288448, + "device": 0, + "inactive_split_bytes_current": 6611586048, + "inactive_split_bytes_peak": 9074092544, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:35:57.532694Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6905187328, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 9074092544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106583556096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T133558543082Z.pickle", + "timestamp_utc": "2026-08-01T13:35:58.649779Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T133558543082Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T133558543082Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..40de0530d2aad29af5be284bd5419f5e4bcdf071 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T133558543082Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eebe4b1cb88486712215b3cf35a82ef4d9a67f241330054a4879d0d49b87254b +size 6067896 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T133559668358Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T133559668358Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f8fa44962862ff4890574944ac157573d8df3bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T133559668358Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104469897216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469897216, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469897216, + "device": 0, + "inactive_split_bytes_current": 6494145536, + "inactive_split_bytes_peak": 9074092544, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:35:59.667933Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469897216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469897216, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469897216, + "device": 0, + "inactive_split_bytes_current": 6905187328, + "inactive_split_bytes_peak": 9074092544, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:35:58.665517Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6494145536, + "inactive_split_bytes_current_delta": -411041792, + "inactive_split_bytes_peak": 9074092544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106583556096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T133559668358Z.pickle", + "timestamp_utc": "2026-08-01T13:35:59.770778Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T133559668358Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T133559668358Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d0ccd41539b8b2cbcdb56fda0a0f991bf5208dfd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T133559668358Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:802293f9f652809c07f604c252ddc6c516c4f98a08f5af722b95088bab6e9e36 +size 5942412 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T133600958714Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T133600958714Z.json new file mode 100644 index 0000000000000000000000000000000000000000..72f48c7b7f43a1063bdaa45d85dff6c4e5462582 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T133600958714Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104469897216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469897216, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469897216, + "device": 0, + "inactive_split_bytes_current": 6905187328, + "inactive_split_bytes_peak": 9074092544, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:36:00.958304Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469897216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469897216, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469897216, + "device": 0, + "inactive_split_bytes_current": 6494145536, + "inactive_split_bytes_peak": 9074092544, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:35:59.783750Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6905187328, + "inactive_split_bytes_current_delta": 411041792, + "inactive_split_bytes_peak": 9074092544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106583556096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T133600958714Z.pickle", + "timestamp_utc": "2026-08-01T13:36:01.058790Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T133600958714Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T133600958714Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..374cf424bcfcb4d070940a4667a53838e630a1f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T133600958714Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90876e02c97c47d548f61fe18995401f5a4b5875f3b6371be30c3e918d9f4c10 +size 6067938 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T133602071937Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T133602071937Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b7de75f6cc086aec6336cd605fbe74d058b45ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T133602071937Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104469897216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469897216, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469897216, + "device": 0, + "inactive_split_bytes_current": 6494145536, + "inactive_split_bytes_peak": 9074092544, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:36:02.071534Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469897216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469897216, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469897216, + "device": 0, + "inactive_split_bytes_current": 6905187328, + "inactive_split_bytes_peak": 9074092544, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:36:01.071824Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6494145536, + "inactive_split_bytes_current_delta": -411041792, + "inactive_split_bytes_peak": 9074092544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106583556096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T133602071937Z.pickle", + "timestamp_utc": "2026-08-01T13:36:02.170535Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T133602071937Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T133602071937Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a4ca57b90587d7a9703059913e0aef928f7cd883 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T133602071937Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5ff6cb177bd9bdac8f3e44dd2f642013a95afa47e0129a2417b89bd54acf663 +size 5942418 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T133653475157Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T133653475157Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8af908d7196b23158c9e08aca248fc499458ba71 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T133653475157Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166269440, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 104469897216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166269440, + "active_bytes_peak": 104469897216, + "allocated_bytes_current": 61166269440, + "allocated_bytes_peak": 104469897216, + "device": 0, + "inactive_split_bytes_current": 7140068352, + "inactive_split_bytes_peak": 9074092544, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:36:53.474720Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166269440, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104469897216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 104469897216, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 104469897216, + "device": 0, + "inactive_split_bytes_current": 6494145536, + "inactive_split_bytes_peak": 9074092544, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_peak": 106583556096, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:36:02.183356Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7140068352, + "inactive_split_bytes_current_delta": 645922816, + "inactive_split_bytes_peak": 9074092544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 106583556096, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106583556096, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T133653475157Z.pickle", + "timestamp_utc": "2026-08-01T13:36:53.670704Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T133653475157Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T133653475157Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1e50bf1c5764901911f7f42de422ab421b2ed9e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T133653475157Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c93233d533ce01aae9e123fc7cd6bf84d79ce9034f70b04eb1280b2e0d83050a +size 9695021 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T133552447143Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T133552447143Z.json new file mode 100644 index 0000000000000000000000000000000000000000..85b1791cba2fe53d5794ea53e5f2c8916dd037aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T133552447143Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 80833020928, + "active_bytes_peak_delta": 45366684672, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 80833020928, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80833020928, + "device": 1, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_peak": 8696209408, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.446562Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 80833020928, + "allocated_bytes_peak_delta": 45366684672, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:34:43.477538Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_current_delta": 5887878144, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 8535823360, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 91802828800, + "reserved_bytes_current_delta": 52558823424, + "reserved_bytes_peak": 91802828800, + "reserved_bytes_peak_delta": 52558823424, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T133552447143Z.pickle", + "timestamp_utc": "2026-08-01T13:35:52.567980Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T133552447143Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T133552447143Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1a6c56d0d0129c3fafad02b14e4d77e6a6ebd5d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T133552447143Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f3da47fbcd736a9bd23a3cfb87b980479b321c0c9d6db440da8bf21f7cae22b +size 5924407 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T133553571295Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T133553571295Z.json new file mode 100644 index 0000000000000000000000000000000000000000..140c95e4cfc53cfbf4121d3391c508d37a6e95e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T133553571295Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104474520064, + "active_bytes_peak_delta": 23641499136, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104474520064, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104474520064, + "device": 1, + "inactive_split_bytes_current": 7547423744, + "inactive_split_bytes_peak": 8696209408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:53.570887Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104474520064, + "allocated_bytes_peak_delta": 23641499136, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 80833020928, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80833020928, + "device": 1, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_peak": 8696209408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.585198Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7547423744, + "inactive_split_bytes_current_delta": 1526726656, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106633887744, + "reserved_bytes_current_delta": 14831058944, + "reserved_bytes_peak": 106633887744, + "reserved_bytes_peak_delta": 14831058944, + "segment_count_current": 486, + "segment_count_current_delta": 65, + "segment_count_peak": 486, + "segment_count_peak_delta": 65, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T133553571295Z.pickle", + "timestamp_utc": "2026-08-01T13:35:53.687183Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T133553571295Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T133553571295Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03a0a4ee2b546346e1f0cbc49233ca3eb0d042de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T133553571295Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8c59df1a72b3e1e8d42bf548f6431b81281c05270f39bda26cba8312508b8f9 +size 5928977 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T133554685250Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T133554685250Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4efd7dbb02b10e439e5550f2ca3e33dc21fedff2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T133554685250Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 2203648, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:54.684862Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 2203648, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104474520064, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104474520064, + "device": 1, + "inactive_split_bytes_current": 7547423744, + "inactive_split_bytes_peak": 8696209408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:53.701562Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_current_delta": -322961408, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 487, + "segment_count_current_delta": 1, + "segment_count_peak": 487, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T133554685250Z.pickle", + "timestamp_utc": "2026-08-01T13:35:54.790100Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T133554685250Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T133554685250Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..02fe2de3fca12ce8ac76b1b391cd067ab2f7c187 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T133554685250Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:764c09e284903402b52b8655eaa7b7d829ba829d9f9612f6d4010bbe59154ac1 +size 5923758 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T133556042407Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T133556042407Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b9e0cdd69360f7c86f2b78b7cadfb8f5a88fdf5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T133556042407Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:56.042004Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:54.803728Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T133556042407Z.pickle", + "timestamp_utc": "2026-08-01T13:35:56.151669Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T133556042407Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T133556042407Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..983517d969db87edbe0e54bdb06ecabc09bdf6bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T133556042407Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35e81933d8e9aab17f3f48b089b2240bb487f9929f0c0e0389e13a5703be161a +size 5923205 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T133557413127Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T133557413127Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee2bb12ff5d2fcad35a557652be47433148d35a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T133557413127Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7283182592, + "inactive_split_bytes_peak": 8696209408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:57.412720Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:56.165766Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7283182592, + "inactive_split_bytes_current_delta": 58720256, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T133557413127Z.pickle", + "timestamp_utc": "2026-08-01T13:35:57.521720Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T133557413127Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T133557413127Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ee27c73467695f068cae7cb43d02564e45a2a146 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T133557413127Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40e729faae3fa5b58224e29f7ff6d4e70a14b41af9298d854bcf9f4d62c3030c +size 5923308 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T133558543101Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T133558543101Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a5c454d246a69495db5d945e8bc68f1b3640e92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T133558543101Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_peak": 9383162880, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:58.542704Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7283182592, + "inactive_split_bytes_peak": 8696209408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:57.536139Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 686953472, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T133558543101Z.pickle", + "timestamp_utc": "2026-08-01T13:35:58.646901Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T133558543101Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T133558543101Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cec6394cd618139a92c48b52ef0c1aa4c10b4039 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T133558543101Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a86dd130bc6caa2165e41453328ad5d887c5220742c67843a547bd8dc28c6cf3 +size 5922706 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T133559668339Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T133559668339Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca1573d17ff66a76419c0709f450a6436b9e9ecc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T133559668339Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 6901500928, + "inactive_split_bytes_peak": 9383162880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:59.667934Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_peak": 9383162880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:58.660383Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6901500928, + "inactive_split_bytes_current_delta": -616562688, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T133559668339Z.pickle", + "timestamp_utc": "2026-08-01T13:35:59.772245Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T133559668339Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T133559668339Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb02ebdeabf98c0da8bbbe1c2dc2d469aae1cced --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T133559668339Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5d2f5a1ecd5f0eccb159081ce3aaf7f969ea3c0aebd248da6d1f6aaf012049f +size 5922082 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T133600958729Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T133600958729Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9f65043fa820e5003dcabd4ec8e37b9cfc71314 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T133600958729Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 9383162880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:00.958325Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 6901500928, + "inactive_split_bytes_peak": 9383162880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:59.785653Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_current_delta": 440401920, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T133600958729Z.pickle", + "timestamp_utc": "2026-08-01T13:36:01.061547Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T133600958729Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T133600958729Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..70bebeb32c8fab0e22799237ca2509b0529d5f76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T133600958729Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91f409c42bdb318505a4e1c4250b652fd244feac55d8428142e23956746149ce +size 5921962 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T133602071938Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T133602071938Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ae32a21719b153210cad0b0e1eb2163aed24b13 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T133602071938Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7400623104, + "inactive_split_bytes_peak": 9383162880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:02.071542Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 9383162880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:01.074769Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7400623104, + "inactive_split_bytes_current_delta": 58720256, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T133602071938Z.pickle", + "timestamp_utc": "2026-08-01T13:36:02.172180Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T133602071938Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T133602071938Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..33d3289670306a22d5e46ef11d6a42e4cc9fe35e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T133602071938Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61c1da881aef8da8b19d6d0dda7c384ddbcef35c6a02e193d6f0e33c99220048 +size 5921931 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T133653475194Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T133653475194Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1583b52f452b1ad67e9062a89605c7097350e856 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T133653475194Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7664864256, + "inactive_split_bytes_peak": 9383162880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:53.474718Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 1, + "inactive_split_bytes_current": 7400623104, + "inactive_split_bytes_peak": 9383162880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:02.185205Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7664864256, + "inactive_split_bytes_current_delta": 264241152, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T133653475194Z.pickle", + "timestamp_utc": "2026-08-01T13:36:53.675755Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T133653475194Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T133653475194Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea2ee3424ba20875657c4109ccf18fe568cefa86 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T133653475194Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c35761adb6bf9976e865b196d793c8541309b4b6ddc0ec9f1298bb32371241a2 +size 9693134 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T133552446918Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T133552446918Z.json new file mode 100644 index 0000000000000000000000000000000000000000..37fc713de8bd9436fee3253a330acd2556ea7bab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T133552446918Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 80833020928, + "active_bytes_peak_delta": 45366684672, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 80833020928, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80833020928, + "device": 2, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_peak": 8696209408, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.446324Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 80833020928, + "allocated_bytes_peak_delta": 45366684672, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:34:43.477515Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_current_delta": 5887878144, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 8535823360, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 91802828800, + "reserved_bytes_current_delta": 52558823424, + "reserved_bytes_peak": 91802828800, + "reserved_bytes_peak_delta": 52558823424, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T133552446918Z.pickle", + "timestamp_utc": "2026-08-01T13:35:52.566720Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T133552446918Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T133552446918Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df6891106025742aee4f3bb2802cbf7e521c64c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T133552446918Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5a085f2d1d7cd84afd7c660e068ccbacc93d84cfe5591ee8240d7c7d7fa2a1f +size 5924409 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T133553571282Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T133553571282Z.json new file mode 100644 index 0000000000000000000000000000000000000000..224518e60a5bb1c83dd42abe1b07694155f29388 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T133553571282Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104474520064, + "active_bytes_peak_delta": 23641499136, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104474520064, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104474520064, + "device": 2, + "inactive_split_bytes_current": 7547423744, + "inactive_split_bytes_peak": 8696209408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:53.570868Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104474520064, + "allocated_bytes_peak_delta": 23641499136, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 80833020928, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80833020928, + "device": 2, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_peak": 8696209408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.583360Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7547423744, + "inactive_split_bytes_current_delta": 1526726656, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106633887744, + "reserved_bytes_current_delta": 14831058944, + "reserved_bytes_peak": 106633887744, + "reserved_bytes_peak_delta": 14831058944, + "segment_count_current": 486, + "segment_count_current_delta": 65, + "segment_count_peak": 486, + "segment_count_peak_delta": 65, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T133553571282Z.pickle", + "timestamp_utc": "2026-08-01T13:35:53.686986Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T133553571282Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T133553571282Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..51bf00d41e348d2acb992323271ee212c42f2cef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T133553571282Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8afc1ba43d43563c488c9d5cd18df08dc94cdb37e4b63b579876db2cc9739032 +size 5928970 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T133554685288Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T133554685288Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d3b25585b256a779275fdd26c3d823aa51dbe0a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T133554685288Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 2203648, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:54.684861Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 2203648, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104474520064, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104474520064, + "device": 2, + "inactive_split_bytes_current": 7547423744, + "inactive_split_bytes_peak": 8696209408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:53.703249Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_current_delta": -322961408, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 487, + "segment_count_current_delta": 1, + "segment_count_peak": 487, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T133554685288Z.pickle", + "timestamp_utc": "2026-08-01T13:35:54.792312Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T133554685288Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T133554685288Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2589e180eaf74c86f1f640a010604493680d05ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T133554685288Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c8ef822577e81c1348579f046787eb3c4fcd673699c9863000d5cd5a7c93d10 +size 5923759 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T133556042371Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T133556042371Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ef1bf5a62fdbea90d99a8c8e882ae31481218f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T133556042371Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:56.041969Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:54.805868Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T133556042371Z.pickle", + "timestamp_utc": "2026-08-01T13:35:56.149963Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T133556042371Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T133556042371Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..37e394bc7d19797b80aa191a0f9860e16f90e2f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T133556042371Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:075bfbe58be24bb58176067a83b417e48e1a2a194ebdc12e59d74227851c66dd +size 5923233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T133557413094Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T133557413094Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cb2823c83df3ce9f02e50806966f332717676640 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T133557413094Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7283182592, + "inactive_split_bytes_peak": 8696209408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:57.412689Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:56.163827Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7283182592, + "inactive_split_bytes_current_delta": 58720256, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T133557413094Z.pickle", + "timestamp_utc": "2026-08-01T13:35:57.521140Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T133557413094Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T133557413094Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..24f210238fa34cd377353a909abd4fc8b1cb0748 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T133557413094Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e1b2bf8493559f873cf1cbf505a66fb2f702dd0d9e817c9ca6ede5ada0885ef +size 5923304 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T133558543061Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T133558543061Z.json new file mode 100644 index 0000000000000000000000000000000000000000..062756db4a5bbf0f454d19dbf8c7b5320fb59043 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T133558543061Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_peak": 9383162880, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:58.542669Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7283182592, + "inactive_split_bytes_peak": 8696209408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:57.534825Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 686953472, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T133558543061Z.pickle", + "timestamp_utc": "2026-08-01T13:35:58.652744Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T133558543061Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T133558543061Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93cbc2c83058ef12649bc128361a3790fcbb35cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T133558543061Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6fa22af6729066c99b31352fde0fe3b186a8ef7559970192d6ac017c9fee5fe +size 5922704 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T133559668324Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T133559668324Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b769bc358ac732c4c61ee4bf88b2f5d1302458ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T133559668324Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 6901500928, + "inactive_split_bytes_peak": 9383162880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:59.667911Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_peak": 9383162880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:58.668941Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6901500928, + "inactive_split_bytes_current_delta": -616562688, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T133559668324Z.pickle", + "timestamp_utc": "2026-08-01T13:35:59.775687Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T133559668324Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T133559668324Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0b3ebe360f22e960b3f030faa6feb1fe8da60e6c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T133559668324Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ca9ae8c9260763b754577f0b3270d3f1f5caa2e36b92cc07cb18bedb6582ece +size 5922078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T133600958707Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T133600958707Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd144b7dbf8e0169f9922d8f3a6c2a3019ea31b4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T133600958707Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 9383162880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:00.958306Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 6901500928, + "inactive_split_bytes_peak": 9383162880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:59.789569Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_current_delta": 440401920, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T133600958707Z.pickle", + "timestamp_utc": "2026-08-01T13:36:01.064131Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T133600958707Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T133600958707Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..410ae1ed0ea35516bb3dd5764cd9509db4e90a5a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T133600958707Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b3c4496b3d9aef74ee9a0564b4a07e57d70b57203539d8703b2ffe22075108e +size 5921964 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T133602071886Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T133602071886Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1adbdf20a3b0f105e54f89f8a3c10c6af5a138a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T133602071886Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7402720256, + "inactive_split_bytes_peak": 9383162880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:02.071494Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 9383162880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:01.077293Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7402720256, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T133602071886Z.pickle", + "timestamp_utc": "2026-08-01T13:36:02.173759Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T133602071886Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T133602071886Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b31a81aaff9516076a7b5cade3cbc5aae6de03ef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T133602071886Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57adf05854f7b281d3bd78f1907f848b9dd03d9feb7db47e665232cdffc4819f +size 5921971 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T133653475332Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T133653475332Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4adee6221f55e5d0ce947eb93ebb380607e5bab5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T133653475332Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7666961408, + "inactive_split_bytes_peak": 9383162880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:53.474746Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 2, + "inactive_split_bytes_current": 7402720256, + "inactive_split_bytes_peak": 9383162880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:02.186823Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7666961408, + "inactive_split_bytes_current_delta": 264241152, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T133653475332Z.pickle", + "timestamp_utc": "2026-08-01T13:36:53.685495Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T133653475332Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T133653475332Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c1ede77bf948fb5cda56f0f0eef21e37d29b5105 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T133653475332Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37bcb477279f919442d3bd2a8cb035693d8ac9e2bdd07dd18b7cfdffb993f070 +size 9693169 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T133552446967Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T133552446967Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f1cbb31717891f3ce216669ab5516744566c22a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T133552446967Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 80833020928, + "active_bytes_peak_delta": 45366684672, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 80833020928, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80833020928, + "device": 3, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_peak": 8696209408, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.446351Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 80833020928, + "allocated_bytes_peak_delta": 45366684672, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:34:43.477548Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_current_delta": 5887878144, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 8535823360, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 91802828800, + "reserved_bytes_current_delta": 52558823424, + "reserved_bytes_peak": 91802828800, + "reserved_bytes_peak_delta": 52558823424, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T133552446967Z.pickle", + "timestamp_utc": "2026-08-01T13:35:52.561768Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T133552446967Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T133552446967Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3919e0aee1edcbc125d50a28cfe4e1512bf48174 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T133552446967Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9e8665df40b7db1084df1d0509abfa333495e586a0a9ff84231c47b02d10f2b +size 5924416 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T133553571264Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T133553571264Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd09a83194284a1de26c4f77bbeaea0f9e5b9c11 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T133553571264Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104474520064, + "active_bytes_peak_delta": 23641499136, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104474520064, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104474520064, + "device": 3, + "inactive_split_bytes_current": 7547423744, + "inactive_split_bytes_peak": 8696209408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:53.570864Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104474520064, + "allocated_bytes_peak_delta": 23641499136, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 80833020928, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80833020928, + "device": 3, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_peak": 8696209408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.576977Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7547423744, + "inactive_split_bytes_current_delta": 1526726656, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 14833156096, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 14833156096, + "segment_count_current": 487, + "segment_count_current_delta": 66, + "segment_count_peak": 487, + "segment_count_peak_delta": 66, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T133553571264Z.pickle", + "timestamp_utc": "2026-08-01T13:35:53.683667Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T133553571264Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T133553571264Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d15ddd0a84a52d01421c55db4e7a9e57932741cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T133553571264Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb63d0b98f305abe1de671c28426db462279f693bd006472817e31d6c87fb9ee +size 5929132 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T133554685242Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T133554685242Z.json new file mode 100644 index 0000000000000000000000000000000000000000..40489ed6a37db21b4950d88344a2774c2833b9c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T133554685242Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 2203648, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:54.684844Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 2203648, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104474520064, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104474520064, + "device": 3, + "inactive_split_bytes_current": 7547423744, + "inactive_split_bytes_peak": 8696209408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:53.695709Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_current_delta": -322961408, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T133554685242Z.pickle", + "timestamp_utc": "2026-08-01T13:35:54.788968Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T133554685242Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T133554685242Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0ee310bbcf06fd580f4d266d4279dcb6f264a40 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T133554685242Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b43f76f0e4cd1bad5d8e637fb2aa3237910c1b0b44b3df4d62dd7963df542064 +size 5923701 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T133556042365Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T133556042365Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3dd588cdb3849aa95b6e44e6397e86ec2d5a6365 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T133556042365Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:56.041967Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:54.800740Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T133556042365Z.pickle", + "timestamp_utc": "2026-08-01T13:35:56.149580Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T133556042365Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T133556042365Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e39c507c94d314f07a2915e26782b13d3b69070e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T133556042365Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f4e3c553f9a95dbc44c1f7d04e1bee81ae24c142189f541a4f490c575112c02 +size 5923238 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T133557413099Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T133557413099Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc8e7bf2b8b80c58efcfc1f4c5d801afbc63423c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T133557413099Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7283182592, + "inactive_split_bytes_peak": 8696209408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:57.412695Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:56.161455Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7283182592, + "inactive_split_bytes_current_delta": 58720256, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T133557413099Z.pickle", + "timestamp_utc": "2026-08-01T13:35:57.523061Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T133557413099Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T133557413099Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b3c23ea62cc4a167b51437756358e15b686fda4e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T133557413099Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8144f9b6490c0a806c3da7c55e9dbc2e57564d1eeeb1f30822bbb435fba22a5 +size 5923308 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T133558543063Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T133558543063Z.json new file mode 100644 index 0000000000000000000000000000000000000000..366a3e0cfe08f35f742063d0d51fe528781593ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T133558543063Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_peak": 9383162880, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:58.542666Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7283182592, + "inactive_split_bytes_peak": 8696209408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:57.537690Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 686953472, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T133558543063Z.pickle", + "timestamp_utc": "2026-08-01T13:35:58.641474Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T133558543063Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T133558543063Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e93908336185e2324fcfa6fb278e48f632c75a67 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T133558543063Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d721bab5854dbd5cc245165a9168a9c14e4b35c69efcf16fea95a7f7cc70953 +size 5922731 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T133559668322Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T133559668322Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aaef1f99e502ca576e518c674c3392f082a6b82a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T133559668322Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 6901500928, + "inactive_split_bytes_peak": 9383162880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:59.667893Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_peak": 9383162880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:58.652993Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6901500928, + "inactive_split_bytes_current_delta": -616562688, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T133559668322Z.pickle", + "timestamp_utc": "2026-08-01T13:35:59.773494Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T133559668322Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T133559668322Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11a004c4c52faf2538fea89cbcf18a4ac6786018 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T133559668322Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4f1d9a41fdd1499a7a038c3e031a774e0b9ea3321d4aade8062c1a5dbbf1bd9 +size 5922082 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T133600958700Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T133600958700Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b8b94f85e641f89accab9a5943838b77d950df0f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T133600958700Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 9383162880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:00.958297Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 6901500928, + "inactive_split_bytes_peak": 9383162880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:59.785862Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_current_delta": 440401920, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T133600958700Z.pickle", + "timestamp_utc": "2026-08-01T13:36:01.061320Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T133600958700Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T133600958700Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c82e705cbc90aa0829ebccffa0706234c397d5ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T133600958700Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7221ccb589af2e91ca51dc6a3f0d1d299adeb5f78742da6f87a7678b062c295e +size 5921993 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T133602071951Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T133602071951Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e533d70a7405d0770894b50a17b762014eaf2bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T133602071951Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7400623104, + "inactive_split_bytes_peak": 9383162880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:02.071539Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 9383162880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:01.073005Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7400623104, + "inactive_split_bytes_current_delta": 58720256, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T133602071951Z.pickle", + "timestamp_utc": "2026-08-01T13:36:02.167985Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T133602071951Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T133602071951Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..56174a638f4a90b3554a7494895e1ee2cde7a68a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T133602071951Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcbb2f208e30d91943cb3d233c3f7e741c03f439b725bdf1e81533da0db19792 +size 5921928 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T133653475287Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T133653475287Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5b1bea7ee719791eaf69b09e9ef425048590faa5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T133653475287Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7664864256, + "inactive_split_bytes_peak": 9383162880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:53.474716Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 3, + "inactive_split_bytes_current": 7400623104, + "inactive_split_bytes_peak": 9383162880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:02.179150Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7664864256, + "inactive_split_bytes_current_delta": 264241152, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T133653475287Z.pickle", + "timestamp_utc": "2026-08-01T13:36:53.682186Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T133653475287Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T133653475287Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c890d42627c87849b46835f21d7e41a4a41d9bfe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T133653475287Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d1b6f31a196258c421c7a5b3b72399d25df400d49ff8805bfb5415c95f93195 +size 9693165 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T133552446878Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T133552446878Z.json new file mode 100644 index 0000000000000000000000000000000000000000..53fbdb0d93193ab90c882bd6a0400466cfa67fb9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T133552446878Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442163200, + "active_bytes_current_delta": 37210265600, + "active_bytes_peak": 80834134016, + "active_bytes_peak_delta": 45367355392, + "after": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 80834134016, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 80834134016, + "device": 4, + "inactive_split_bytes_current": 5815160832, + "inactive_split_bytes_peak": 8787344384, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.446305Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442163200, + "allocated_bytes_current_delta": 37210265600, + "allocated_bytes_peak": 80834134016, + "allocated_bytes_peak_delta": 45367355392, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:34:43.477332Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5815160832, + "inactive_split_bytes_current_delta": 5682784256, + "inactive_split_bytes_peak": 8787344384, + "inactive_split_bytes_peak_delta": 8633934848, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 91802828800, + "reserved_bytes_current_delta": 52558823424, + "reserved_bytes_peak": 91802828800, + "reserved_bytes_peak_delta": 52558823424, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T133552446878Z.pickle", + "timestamp_utc": "2026-08-01T13:35:52.563852Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T133552446878Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T133552446878Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f3736b0352e5f6db3da1f33386f8bc9e7a33f68 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T133552446878Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa10cd4a70d1318e4a385a7cefbf23203856218b20919fe55cb9d8bf7946cd73 +size 5933029 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T133553571263Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T133553571263Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61334c8617da19f71892e0298c2e4aada7f14a2c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T133553571263Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442163200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471729664, + "active_bytes_peak_delta": 23637595648, + "after": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471729664, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471729664, + "device": 4, + "inactive_split_bytes_current": 6754684928, + "inactive_split_bytes_peak": 8787344384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:53.570841Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442163200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471729664, + "allocated_bytes_peak_delta": 23637595648, + "before": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 80834134016, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 80834134016, + "device": 4, + "inactive_split_bytes_current": 5815160832, + "inactive_split_bytes_peak": 8787344384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.582821Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6754684928, + "inactive_split_bytes_current_delta": 939524096, + "inactive_split_bytes_peak": 8787344384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106342383616, + "reserved_bytes_current_delta": 14539554816, + "reserved_bytes_peak": 106342383616, + "reserved_bytes_peak_delta": 14539554816, + "segment_count_current": 486, + "segment_count_current_delta": 65, + "segment_count_peak": 486, + "segment_count_peak_delta": 65, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T133553571263Z.pickle", + "timestamp_utc": "2026-08-01T13:35:53.689123Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T133553571263Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T133553571263Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..439cdca871a014739c84fddf63e411dfb72570df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T133553571263Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dd18a04176362ff0d43a414a5e79f7e87bbe8e294764e7e9d7e088cafe375f1 +size 6067544 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T133554685244Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T133554685244Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6c42863f36f13a3d2e8890fbae4e0a3e5b5ff45 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T133554685244Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442163200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471729664, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471729664, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471729664, + "device": 4, + "inactive_split_bytes_current": 6725324800, + "inactive_split_bytes_peak": 8787344384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:54.684834Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442163200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471729664, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471729664, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471729664, + "device": 4, + "inactive_split_bytes_current": 6754684928, + "inactive_split_bytes_peak": 8787344384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:53.707236Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6725324800, + "inactive_split_bytes_current_delta": -29360128, + "inactive_split_bytes_peak": 8787344384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106342383616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106342383616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T133554685244Z.pickle", + "timestamp_utc": "2026-08-01T13:35:54.793839Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T133554685244Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T133554685244Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..abc26bbbb751f632aad53257b54b24e443d35f16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T133554685244Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4750736c9eb9e3ecee02565c3082abe4ca5c724bbd122e7c76c3b273ab5cccc5 +size 6063521 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T133556042397Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T133556042397Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ffcf41f29069e39928317952cf72bcd926067af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T133556042397Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442163200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471992320, + "active_bytes_peak_delta": 262656, + "after": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_peak": 8787344384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:56.041969Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442163200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471992320, + "allocated_bytes_peak_delta": 262656, + "before": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471729664, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471729664, + "device": 4, + "inactive_split_bytes_current": 6725324800, + "inactive_split_bytes_peak": 8787344384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:54.809630Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_current_delta": -413138944, + "inactive_split_bytes_peak": 8787344384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106342383616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106342383616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T133556042397Z.pickle", + "timestamp_utc": "2026-08-01T13:35:56.152323Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T133556042397Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T133556042397Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fc556bb03ce1612c66418d227eb748ff012a0c7e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T133556042397Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f9ecd74cadbfeedd64f74362fe37b8b79311a6d8a1f5120f12e8e32b65f7de1 +size 6054320 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T133557413130Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T133557413130Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd49557cbe9df642c15abd1f5735b90ba07a6d03 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T133557413130Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442163200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471992320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_peak": 8787344384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:57.412694Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442163200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471992320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_peak": 8787344384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:56.167755Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8787344384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106342383616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106342383616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T133557413130Z.pickle", + "timestamp_utc": "2026-08-01T13:35:57.526115Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T133557413130Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T133557413130Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..211a2377d02ab326dbf973a786e0dff7bf8e2e9a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T133557413130Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21dcca3bd455147f3521a4011c53ef342d0647c19de4676cfecd8b33b7f2014d +size 6054328 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T133558543061Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T133558543061Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bcce8011de3690a3d2a6ce758005af4428b7ac0b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T133558543061Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442163200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471992320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_peak": 8787344384, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:58.542646Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442163200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471992320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_peak": 8787344384, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:57.541688Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8787344384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106342383616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106342383616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T133558543061Z.pickle", + "timestamp_utc": "2026-08-01T13:35:58.650528Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T133558543061Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T133558543061Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9a7150ebae196bdc12ac476b0cb2b444eb7c8dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T133558543061Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d3bdf6218843496518ca39758b70c29d61c501360df2c1fcf027b8102ee13d4 +size 6054319 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T133559668341Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T133559668341Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c64d6855864080755a8016b52a9e5ab8ca64c165 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T133559668341Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442163200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471992320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_peak": 8787344384, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:59.667927Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442163200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471992320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_peak": 8787344384, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:58.666231Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8787344384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106342383616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106342383616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T133559668341Z.pickle", + "timestamp_utc": "2026-08-01T13:35:59.775720Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T133559668341Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T133559668341Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e5732aa4c11da2abff16754e6b54a6337f8be46c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T133559668341Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb383f2657e7123c48ad18848d0e67d12b3c38fbde21c0314dadfc94491575fa +size 6054328 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T133600958707Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T133600958707Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ff26511f8f96355a2656a0c0a682f6ba1ab788a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T133600958707Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442163200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471992320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_peak": 8787344384, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:36:00.958284Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442163200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471992320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_peak": 8787344384, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:59.790839Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8787344384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106342383616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106342383616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T133600958707Z.pickle", + "timestamp_utc": "2026-08-01T13:36:01.065086Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T133600958707Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T133600958707Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..115fca616be12db77427d3a42c6a8a2cfb1c63dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T133600958707Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18c000ae955bef95869224a63cd33f716d2b2b312a0b8733bd006e715bc78351 +size 6054328 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T133602071936Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T133602071936Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7b523fec7fd212b7c55b9a4904039b4268e2c67b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T133602071936Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442163200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471992320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_peak": 8787344384, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:36:02.071504Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442163200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471992320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_peak": 8787344384, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:36:01.080526Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8787344384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106342383616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106342383616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T133602071936Z.pickle", + "timestamp_utc": "2026-08-01T13:36:02.178071Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T133602071936Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T133602071936Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6e0b2624cb00f3eb8d4659334faabaae24d674a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T133602071936Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5673ad88803a09de448d5e902d0179f0d49baeb18cd7531e23c566dce5588b91 +size 6054328 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T133653475192Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T133653475192Z.json new file mode 100644 index 0000000000000000000000000000000000000000..83e49029ee1d970b1c2ce49746e0f9afc7c41a81 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T133653475192Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61167874048, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 104471992320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61167874048, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 61167874048, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6635147264, + "inactive_split_bytes_peak": 8787344384, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:36:53.474719Z", + "world_size": 8 + }, + "allocated_bytes_current": 61167874048, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104471992320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442163200, + "active_bytes_peak": 104471992320, + "allocated_bytes_current": 72442163200, + "allocated_bytes_peak": 104471992320, + "device": 4, + "inactive_split_bytes_current": 6312185856, + "inactive_split_bytes_peak": 8787344384, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 106342383616, + "reserved_bytes_peak": 106342383616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:36:02.193562Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6635147264, + "inactive_split_bytes_current_delta": 322961408, + "inactive_split_bytes_peak": 8787344384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 106342383616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106342383616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T133653475192Z.pickle", + "timestamp_utc": "2026-08-01T13:36:53.677141Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T133653475192Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T133653475192Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..631b4b759330d91fc7c00e84093cb57ffa918250 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T133653475192Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1b91eed6b10756d8307860a636c8103203cf3b3918c108f23f4eb73c1637a5d +size 9696850 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T133552446978Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T133552446978Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c42c259e28d8089343a651a3e36367201abe9b4a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T133552446978Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 80833020928, + "active_bytes_peak_delta": 45366684672, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 80833020928, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80833020928, + "device": 5, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_peak": 8696209408, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.446372Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 80833020928, + "allocated_bytes_peak_delta": 45366684672, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:34:43.477568Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_current_delta": 5887878144, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 8535823360, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 91802828800, + "reserved_bytes_current_delta": 52558823424, + "reserved_bytes_peak": 91802828800, + "reserved_bytes_peak_delta": 52558823424, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T133552446978Z.pickle", + "timestamp_utc": "2026-08-01T13:35:52.563543Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T133552446978Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T133552446978Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ececeb4ab8e77e473761aebb02a1b232b1edc39b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T133552446978Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a6bdf0a642cea87e83e00d97a3a29211d4442d37c69c3ca9a2274e7a9313d26 +size 5924411 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T133553571312Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T133553571312Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d2a32f552f656d57948146d5c7bda84fa413b8db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T133553571312Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471398400, + "active_bytes_peak_delta": 23638377472, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_peak": 8696209408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106604527616, + "reserved_bytes_peak": 106604527616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:53.570894Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471398400, + "allocated_bytes_peak_delta": 23638377472, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 80833020928, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80833020928, + "device": 5, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_peak": 8696209408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.580118Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_current_delta": 1497366528, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106604527616, + "reserved_bytes_current_delta": 14801698816, + "reserved_bytes_peak": 106604527616, + "reserved_bytes_peak_delta": 14801698816, + "segment_count_current": 486, + "segment_count_current_delta": 65, + "segment_count_peak": 486, + "segment_count_peak_delta": 65, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T133553571312Z.pickle", + "timestamp_utc": "2026-08-01T13:35:53.686431Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T133553571312Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T133553571312Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..074e9b10456b6b07ef5a6ba3cb5e8b4b23740f77 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T133553571312Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58848e92f8df25bb5b7163262140ad55700e1881490f3415b600e139206d5b98 +size 5927718 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T133554685280Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T133554685280Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c0e555a35fac05858b0d0ec2464d350d3d82e70c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T133554685280Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471398400, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:54.684875Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471398400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_peak": 8696209408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106604527616, + "reserved_bytes_peak": 106604527616, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:53.700858Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106606624768, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 106606624768, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 487, + "segment_count_current_delta": 1, + "segment_count_peak": 487, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T133554685280Z.pickle", + "timestamp_utc": "2026-08-01T13:35:54.793511Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T133554685280Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T133554685280Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5cbf1aa6bf3985c16ace9ce1b3e0474e45704358 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T133554685280Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25adda277039bc63736c5ae7a72c5096d09fbc6f59990f44dafdbd6e367af404 +size 5923818 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T133556042409Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T133556042409Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0c0faa8fdbb308796a6c944b5e5ebe58f1ef5509 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T133556042409Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471398400, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:56.042008Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471398400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:54.807075Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106606624768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106606624768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T133556042409Z.pickle", + "timestamp_utc": "2026-08-01T13:35:56.150135Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T133556042409Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T133556042409Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb8dfe52a731ec98336070fa11b5cc49f2e69a01 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T133556042409Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9923135a37402c22ab1c14d7287231395e0191e3f9891861a4c6c238abd6121 +size 5923659 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T133557413133Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T133557413133Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d8954916450a7fc95810cc940a28a0807be70cfa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T133557413133Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471398400, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:57.412728Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471398400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:56.164076Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106606624768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106606624768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T133557413133Z.pickle", + "timestamp_utc": "2026-08-01T13:35:57.519180Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T133557413133Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T133557413133Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..276c05d6e1dab25e342705ef5328a1acdc55a2a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T133557413133Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:887b940c3912d67dff74c40495642c2d9833d8434cafc5961b597f62eab243c9 +size 5923651 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T133558543103Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T133558543103Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0063707d3cf7a32e1843b049212cb967a196ebd3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T133558543103Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471398400, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:58.542715Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471398400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:57.532958Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106606624768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106606624768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T133558543103Z.pickle", + "timestamp_utc": "2026-08-01T13:35:58.647601Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T133558543103Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T133558543103Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f55f83b013d87c80b4befd51755f6dc709c26c2e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T133558543103Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae72b41cf6b18e366c864e491bb17822bac093ae9764a55dc3fb24346f68ef72 +size 5923656 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T133559668360Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T133559668360Z.json new file mode 100644 index 0000000000000000000000000000000000000000..07fa4505765dc7b9929f5994d068349db7c2f400 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T133559668360Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471398400, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:59.667939Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471398400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:58.661302Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106606624768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106606624768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T133559668360Z.pickle", + "timestamp_utc": "2026-08-01T13:35:59.780027Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T133559668360Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T133559668360Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6455f268791c638ba39c7ba72b0de6342de50530 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T133559668360Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:effe4a45b5825d2135761c25509040158dfefad2b665ba6f1be00f3b70153b84 +size 5923656 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T133600958731Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T133600958731Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c7820d6d3051eea2f1fab8fe686995aaf7d064d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T133600958731Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471398400, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:00.958323Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471398400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:59.793782Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106606624768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106606624768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T133600958731Z.pickle", + "timestamp_utc": "2026-08-01T13:36:01.066017Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T133600958731Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T133600958731Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e11fe246816f805769782faea32d9ddb5618bb27 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T133600958731Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dab14e7fa6a441960dc310f60b66d4378da688d78c8994d5a5a4ff9624e47d2 +size 5923651 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T133602071941Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T133602071941Z.json new file mode 100644 index 0000000000000000000000000000000000000000..926b70d0af7547ee79714f0e41af14c0aa1b76d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T133602071941Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104471398400, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:02.071540Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471398400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:01.081729Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106606624768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106606624768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T133602071941Z.pickle", + "timestamp_utc": "2026-08-01T13:36:02.173066Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T133602071941Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T133602071941Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f9cad19bfc3024ac165ae15b500a9f7dc26b2c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T133602071941Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59609be58818fbc0f39cd08cd5e5882a6131f167bf5d612e7186cfe00074adf8 +size 5923651 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T133653475286Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T133653475286Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb5352c4d79d39dcc908df7e1ce13ab16493cf03 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T133653475286Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 104471398400, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7664864256, + "inactive_split_bytes_peak": 8696209408, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:53.474717Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104471398400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104471398400, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471398400, + "device": 5, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 8696209408, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 106606624768, + "reserved_bytes_peak": 106606624768, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:02.185880Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7664864256, + "inactive_split_bytes_current_delta": 322961408, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 106606624768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106606624768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T133653475286Z.pickle", + "timestamp_utc": "2026-08-01T13:36:53.674979Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T133653475286Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T133653475286Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d105fb0729583014c412d52eaee42224c14a7969 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T133653475286Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccd8c9cc7bfbd0a0b642215fd766971b548faa021164511d3d9e00116e027b37 +size 9694912 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T133552446930Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T133552446930Z.json new file mode 100644 index 0000000000000000000000000000000000000000..269aa9d6aa3f0ca80a38bde0ee33b4547313c296 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T133552446930Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 80833020928, + "active_bytes_peak_delta": 45366684672, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 80833020928, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80833020928, + "device": 6, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_peak": 8696209408, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.446323Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 80833020928, + "allocated_bytes_peak_delta": 45366684672, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:34:43.477497Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_current_delta": 5887878144, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 8535823360, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 91802828800, + "reserved_bytes_current_delta": 52558823424, + "reserved_bytes_peak": 91802828800, + "reserved_bytes_peak_delta": 52558823424, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T133552446930Z.pickle", + "timestamp_utc": "2026-08-01T13:35:52.566019Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T133552446930Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T133552446930Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dea60c9ea3f02bcc396db708889a4db5202f7756 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T133552446930Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d302c5bae055c8f08b951ff9bad86e68271c246bc076dcd4c50d51880b08c4e +size 5924404 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T133553571354Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T133553571354Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e2e1e83bed85819e975a46b1fbb86a9b2140d982 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T133553571354Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104474520064, + "active_bytes_peak_delta": 23641499136, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104474520064, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104474520064, + "device": 6, + "inactive_split_bytes_current": 7547423744, + "inactive_split_bytes_peak": 8696209408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:53.570931Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104474520064, + "allocated_bytes_peak_delta": 23641499136, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 80833020928, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80833020928, + "device": 6, + "inactive_split_bytes_current": 6020697088, + "inactive_split_bytes_peak": 8696209408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.584732Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7547423744, + "inactive_split_bytes_current_delta": 1526726656, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106633887744, + "reserved_bytes_current_delta": 14831058944, + "reserved_bytes_peak": 106633887744, + "reserved_bytes_peak_delta": 14831058944, + "segment_count_current": 486, + "segment_count_current_delta": 65, + "segment_count_peak": 486, + "segment_count_peak_delta": 65, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T133553571354Z.pickle", + "timestamp_utc": "2026-08-01T13:35:53.688319Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T133553571354Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T133553571354Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..62de94903bfaf6cddd55da72fb89f472e1c8f408 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T133553571354Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b00012f86aaf3d8ac450be64bd67fae10b1b4e61cf6c520093eaf48c14dc9f3c +size 5928976 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T133554685244Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T133554685244Z.json new file mode 100644 index 0000000000000000000000000000000000000000..20ed90c95481321a78fdfdca78d244daae08c81a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T133554685244Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 2203648, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:54.684843Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 2203648, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104474520064, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104474520064, + "device": 6, + "inactive_split_bytes_current": 7547423744, + "inactive_split_bytes_peak": 8696209408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:53.704434Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_current_delta": -322961408, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 487, + "segment_count_current_delta": 1, + "segment_count_peak": 487, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T133554685244Z.pickle", + "timestamp_utc": "2026-08-01T13:35:54.794782Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T133554685244Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T133554685244Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ab1ff71d9f038d529360366137e4eea2bbf1614b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T133554685244Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:633598a5c6100ccd68e35ae1eb147f113d1bf59e574187d4c0ec80f6f69239fb +size 5923761 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T133556042396Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T133556042396Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d9df1e845f1c10ab2944a66884de075c8d10e237 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T133556042396Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:56.041973Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:54.810305Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T133556042396Z.pickle", + "timestamp_utc": "2026-08-01T13:35:56.158536Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T133556042396Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T133556042396Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..02984321f9d35deefc966055214db802cfe80471 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T133556042396Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51d38af6b22af9ab805e5b7b370262816fe58b1242001d416a8717da1a6a73b5 +size 5923234 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T133557413114Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T133557413114Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d2bf10fc29c987512a2f5015d19a1bbaed0b00b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T133557413114Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7283182592, + "inactive_split_bytes_peak": 8696209408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:57.412701Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7224462336, + "inactive_split_bytes_peak": 8696209408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:56.175943Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7283182592, + "inactive_split_bytes_current_delta": 58720256, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T133557413114Z.pickle", + "timestamp_utc": "2026-08-01T13:35:57.527494Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T133557413114Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T133557413114Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7a664c0f64e6e4df0570d9d78127fcfa789789e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T133557413114Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2547aeaae929204a73ce3117b20bea8ec81e451e8ddac9a110344568686d0afa +size 5923308 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T133558543066Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T133558543066Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b02fcb4ad8a23783a39d4f9cbd63ad2d783b6a15 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T133558543066Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_peak": 9383162880, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:58.542674Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7283182592, + "inactive_split_bytes_peak": 8696209408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:57.545576Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 686953472, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T133558543066Z.pickle", + "timestamp_utc": "2026-08-01T13:35:58.651634Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T133558543066Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T133558543066Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b25bfa594ff89944b456b33ea0a140b461eb3396 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T133558543066Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48790dfe522400605fc22203bf37060d2b08373cb9923ad40477e5dd458e8d6f +size 5922731 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T133559668340Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T133559668340Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69b290fe4473cb291f91a090135df60be83f5892 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T133559668340Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 6901500928, + "inactive_split_bytes_peak": 9383162880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:59.667922Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7518063616, + "inactive_split_bytes_peak": 9383162880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:58.667185Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6901500928, + "inactive_split_bytes_current_delta": -616562688, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T133559668340Z.pickle", + "timestamp_utc": "2026-08-01T13:35:59.776112Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T133559668340Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T133559668340Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c31ba9f9cba956b9f8c1484ce0326b0303a198a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T133559668340Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d3cc12878e485456b2322ca0303cafcb88f2df502cbd0f5fc042695ceddd2d +size 5922082 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T133600958709Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T133600958709Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8477a402c4c843aede72150523509eae9622a23f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T133600958709Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 9383162880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:00.958290Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 6901500928, + "inactive_split_bytes_peak": 9383162880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:35:59.791098Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_current_delta": 440401920, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T133600958709Z.pickle", + "timestamp_utc": "2026-08-01T13:36:01.064734Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T133600958709Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T133600958709Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..544230bc670e373f8c1e7133882632fc3903c028 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T133600958709Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:238c648f477a1d0954cd406e93393ec2e4ce7053bd8b861446d3979439de8634 +size 5921993 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T133602071911Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T133602071911Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4731168e2874ea58cbd0b0cb768265a7a0075a03 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T133602071911Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7400623104, + "inactive_split_bytes_peak": 9383162880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:02.071510Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7341902848, + "inactive_split_bytes_peak": 9383162880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:01.079811Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7400623104, + "inactive_split_bytes_current_delta": 58720256, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T133602071911Z.pickle", + "timestamp_utc": "2026-08-01T13:36:02.175518Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T133602071911Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T133602071911Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..afdce52d14d25d5fc91ccaa7b851e37ea5218d6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T133602071911Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:442ac5d1e473350222891dfaef0d06d062ccd419e873387a6004cd4d91fd49d5 +size 5921931 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T133653475267Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T133653475267Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2f8b8b0441f3440724da1ab552c83fbfd25ca04f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T133653475267Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 104476723712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7664864256, + "inactive_split_bytes_peak": 9383162880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:53.474717Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104476723712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104476723712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476723712, + "device": 6, + "inactive_split_bytes_current": 7400623104, + "inactive_split_bytes_peak": 9383162880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 106635984896, + "reserved_bytes_peak": 106635984896, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T13:36:02.190473Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7664864256, + "inactive_split_bytes_current_delta": 264241152, + "inactive_split_bytes_peak": 9383162880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 106635984896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106635984896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T133653475267Z.pickle", + "timestamp_utc": "2026-08-01T13:36:53.686563Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T133653475267Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T133653475267Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13f5210bc9fed6ba393a3480801751123004af41 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T133653475267Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdb55cffd0ee528ee989d5b556cd08a05e2f2a035c3e164927c4e72498f99afc +size 9693165 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T133552446905Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T133552446905Z.json new file mode 100644 index 0000000000000000000000000000000000000000..038f0ac1cd9431e3c044bd21360a60e9a7254c39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T133552446905Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 37208595968, + "active_bytes_peak": 80833020928, + "active_bytes_peak_delta": 45366684672, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 80833020928, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 80833020928, + "device": 7, + "inactive_split_bytes_current": 6020696576, + "inactive_split_bytes_peak": 8696209408, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.446334Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 37208595968, + "allocated_bytes_peak": 80833020928, + "allocated_bytes_peak_delta": 45366684672, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:34:43.477724Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6020696576, + "inactive_split_bytes_current_delta": 5887877632, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 8535823360, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 91802828800, + "reserved_bytes_current_delta": 52558823424, + "reserved_bytes_peak": 91802828800, + "reserved_bytes_peak_delta": 52558823424, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T133552446905Z.pickle", + "timestamp_utc": "2026-08-01T13:35:52.563144Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T133552446905Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T133552446905Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d4d7e75690e180cba8f32360fa921d5cdf2d8ecc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T133552446905Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52ecc8854df6eef1e6615967e6c56988cea04f41686dcf6c59e8650792eab7d1 +size 5926004 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T133553571294Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T133553571294Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac23c46cb94096311b29b7363225b66df6818ccd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T133553571294Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104474520576, + "active_bytes_peak_delta": 23641499648, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104474520576, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474520576, + "device": 7, + "inactive_split_bytes_current": 7547423232, + "inactive_split_bytes_peak": 8696209408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:53.570871Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104474520576, + "allocated_bytes_peak_delta": 23641499648, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 80833020928, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 80833020928, + "device": 7, + "inactive_split_bytes_current": 6020696576, + "inactive_split_bytes_peak": 8696209408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 91802828800, + "reserved_bytes_peak": 91802828800, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:35:52.581706Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7547423232, + "inactive_split_bytes_current_delta": 1526726656, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106633887744, + "reserved_bytes_current_delta": 14831058944, + "reserved_bytes_peak": 106633887744, + "reserved_bytes_peak_delta": 14831058944, + "segment_count_current": 486, + "segment_count_current_delta": 65, + "segment_count_peak": 486, + "segment_count_peak_delta": 65, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T133553571294Z.pickle", + "timestamp_utc": "2026-08-01T13:35:53.688207Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T133553571294Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T133553571294Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff457b3aa776d57f487500eac842992480d20645 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T133553571294Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d024848e617ba27def07d8de4744fa57cdcd3c7aa778548bb4553526c053f07f +size 5930723 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T133554685303Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T133554685303Z.json new file mode 100644 index 0000000000000000000000000000000000000000..947b45e7d10ab2bb8bc9c9a0f06aed9c983943e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T133554685303Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476724224, + "active_bytes_peak_delta": 2203648, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7224461824, + "inactive_split_bytes_peak": 8696209408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:54.684900Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476724224, + "allocated_bytes_peak_delta": 2203648, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104474520576, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474520576, + "device": 7, + "inactive_split_bytes_current": 7547423232, + "inactive_split_bytes_peak": 8696209408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:53.704748Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7224461824, + "inactive_split_bytes_current_delta": -322961408, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106633887744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106633887744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T133554685303Z.pickle", + "timestamp_utc": "2026-08-01T13:35:54.799320Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T133554685303Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T133554685303Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..58998ccf856805a29e0f9bf9c844d439d781ac8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T133554685303Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63f130ecbc2445218b7e75093e11aee8f571d09e1f14d4f60e05d5f4a873af4a +size 5925343 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T133556042497Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T133556042497Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c4a5433840ceeec2c6e7a4d8f2afd3cd3184717d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T133556042497Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476724224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7224461824, + "inactive_split_bytes_peak": 8696209408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:56.042061Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476724224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7224461824, + "inactive_split_bytes_peak": 8696209408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:54.814840Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7224461824, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106633887744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106633887744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T133556042497Z.pickle", + "timestamp_utc": "2026-08-01T13:35:56.156825Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T133556042497Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T133556042497Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3ff98e2819c325f332361937f3402f97f0d933c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T133556042497Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce2f414209ea84cc565a75b39b2f8dfb744dfdc97d332df768abdb077860d928 +size 5924855 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T133557413154Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T133557413154Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bd191bc3ffb6578a378cd12864a0aaf208bb34c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T133557413154Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476724224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7283182080, + "inactive_split_bytes_peak": 8696209408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:57.412747Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476724224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7224461824, + "inactive_split_bytes_peak": 8696209408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:56.172228Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7283182080, + "inactive_split_bytes_current_delta": 58720256, + "inactive_split_bytes_peak": 8696209408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106633887744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106633887744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T133557413154Z.pickle", + "timestamp_utc": "2026-08-01T13:35:57.522369Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T133557413154Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T133557413154Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ac732c347a0562218a6e5b6925a6e0bdcec42eda --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T133557413154Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b2f60b31f5ecec70c3085d37b37a62680854133649997aa56aeca0c8db0636f +size 5924966 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T133558543113Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T133558543113Z.json new file mode 100644 index 0000000000000000000000000000000000000000..54e506ee09d4000cd8346ad617ad0bf5d4e6ab82 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T133558543113Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476724224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7518063104, + "inactive_split_bytes_peak": 9383162368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:58.542714Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476724224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7283182080, + "inactive_split_bytes_peak": 8696209408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:57.537632Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7518063104, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 9383162368, + "inactive_split_bytes_peak_delta": 686952960, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106633887744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106633887744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T133558543113Z.pickle", + "timestamp_utc": "2026-08-01T13:35:58.650328Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T133558543113Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T133558543113Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e38f13187b00c570dfa78dff99fe10cf509ac508 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T133558543113Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c005a07c4e136a4f9835b582576d820b57c1d7165f298bf9e55487ffd45eecc +size 5924389 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T133559668364Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T133559668364Z.json new file mode 100644 index 0000000000000000000000000000000000000000..66119758bc99fa13f91af1aa4b6fb691ef79e138 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T133559668364Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476724224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 6901500416, + "inactive_split_bytes_peak": 9383162368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:59.667954Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476724224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7518063104, + "inactive_split_bytes_peak": 9383162368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:58.665711Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6901500416, + "inactive_split_bytes_current_delta": -616562688, + "inactive_split_bytes_peak": 9383162368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106633887744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106633887744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T133559668364Z.pickle", + "timestamp_utc": "2026-08-01T13:35:59.788148Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T133559668364Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T133559668364Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f23cd52c994c553a0852cf3ecb5ad971d6903df3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T133559668364Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f7f069f2410dc21b1bbfe3b0bba45b0df05ec5613791623a19e5541012ff9df +size 5923708 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T133600958745Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T133600958745Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e94c524d0f37d12192614b22c9a9403e3e2e796 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T133600958745Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476724224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7341902336, + "inactive_split_bytes_peak": 9383162368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:36:00.958342Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476724224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 6901500416, + "inactive_split_bytes_peak": 9383162368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:35:59.806456Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7341902336, + "inactive_split_bytes_current_delta": 440401920, + "inactive_split_bytes_peak": 9383162368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106633887744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106633887744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T133600958745Z.pickle", + "timestamp_utc": "2026-08-01T13:36:01.066045Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T133600958745Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T133600958745Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de3c3f3e4bb2ce26b4b426f01913cce6dbee3d10 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T133600958745Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c689633c0bc47c9395b3d77e94f62f3d509dfd469cf49364a2e026b560966df2 +size 5923650 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T133602071944Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T133602071944Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1b1e927de1620d0c38e85c61be32dee473ed6070 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T133602071944Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104476724224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7400622592, + "inactive_split_bytes_peak": 9383162368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:36:02.071547Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476724224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7341902336, + "inactive_split_bytes_peak": 9383162368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:36:01.081209Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7400622592, + "inactive_split_bytes_current_delta": 58720256, + "inactive_split_bytes_peak": 9383162368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106633887744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106633887744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T133602071944Z.pickle", + "timestamp_utc": "2026-08-01T13:36:02.177896Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T133602071944Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T133602071944Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7204d52b3680d2a9ab4f76011c5709bcf0406082 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T133602071944Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d84e93fb2cb9aace81563485fdb6467ea83d9d5a05d1ee7664cff0ee1132bf9 +size 5923589 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T133653475302Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T133653475302Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2de9c6aa55d35188f5239818653a7b0cf08f442f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T133653475302Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165762048, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 104476724224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165762048, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 61165762048, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7664863744, + "inactive_split_bytes_peak": 9383162368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:36:53.474804Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165762048, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104476724224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 104476724224, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104476724224, + "device": 7, + "inactive_split_bytes_current": 7400622592, + "inactive_split_bytes_peak": 9383162368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 106633887744, + "reserved_bytes_peak": 106633887744, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T13:36:02.195052Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7664863744, + "inactive_split_bytes_current_delta": 264241152, + "inactive_split_bytes_peak": 9383162368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 106633887744, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 106633887744, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T133653475302Z.pickle", + "timestamp_utc": "2026-08-01T13:36:53.688136Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T133653475302Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T133653475302Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ffe86f6f4b46cbeaa1afaa4ff635cd513cc8fb88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T133653475302Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7849e827ba0ffe5d5b1d2af474bddcde203ecc2c39745e25fd99f1a0f100da5e +size 9694795 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..8247844e10f204a43bc066404ec758b4ad8b23cf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 2048 +capacity: 1.25 +expert_dispatch_overlap: on +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6030 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 220 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T133938277176Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T133938277176Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04410993c83d319fd5d1888157bbc268103768d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T133938277176Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854843392, + "active_bytes_current_delta": 31623388160, + "active_bytes_peak": 80919878656, + "active_bytes_peak_delta": 45453542400, + "after": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 80919878656, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 80919878656, + "device": 0, + "inactive_split_bytes_current": 4848880640, + "inactive_split_bytes_peak": 5285151744, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 97538539520, + "reserved_bytes_peak": 97538539520, + "segment_count_current": 513, + "segment_count_peak": 513, + "timestamp_utc": "2026-08-01T13:39:38.276745Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854843392, + "allocated_bytes_current_delta": 31623388160, + "allocated_bytes_peak": 80919878656, + "allocated_bytes_peak_delta": 45453542400, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:38:29.684915Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4848880640, + "inactive_split_bytes_current_delta": 4713964544, + "inactive_split_bytes_peak": 5285151744, + "inactive_split_bytes_peak_delta": 5124519936, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 97538539520, + "reserved_bytes_current_delta": 58292436992, + "reserved_bytes_peak": 97538539520, + "reserved_bytes_peak_delta": 58292436992, + "segment_count_current": 513, + "segment_count_current_delta": 441, + "segment_count_peak": 513, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T133938277176Z.pickle", + "timestamp_utc": "2026-08-01T13:39:38.388504Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T133938277176Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T133938277176Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dbec64173f05627d1f740b96bcea8d6c86d3a36e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T133938277176Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a31b0abf4991b78dd6d1c9184242176d9640b169ccf67ac830519e9da3168fc +size 5395273 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T133939278243Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T133939278243Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c61e3845235144bf3ba5b29a760331c97f3e1bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T133939278243Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854843392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104556275712, + "active_bytes_peak_delta": 23636397056, + "after": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556275712, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556275712, + "device": 0, + "inactive_split_bytes_current": 4630776832, + "inactive_split_bytes_peak": 7646802944, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:39.277849Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854843392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104556275712, + "allocated_bytes_peak_delta": 23636397056, + "before": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 80919878656, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 80919878656, + "device": 0, + "inactive_split_bytes_current": 4848880640, + "inactive_split_bytes_peak": 5285151744, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 97538539520, + "reserved_bytes_peak": 97538539520, + "segment_count_current": 513, + "segment_count_peak": 513, + "timestamp_utc": "2026-08-01T13:39:38.407777Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4630776832, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 7646802944, + "inactive_split_bytes_peak_delta": 2361651200, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_current_delta": 10567548928, + "reserved_bytes_peak": 108106088448, + "reserved_bytes_peak_delta": 10567548928, + "segment_count_current": 557, + "segment_count_current_delta": 44, + "segment_count_peak": 557, + "segment_count_peak_delta": 44, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T133939278243Z.pickle", + "timestamp_utc": "2026-08-01T13:39:39.390447Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T133939278243Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T133939278243Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..722d3f2304473818b4beb7b6259f43bfc539b960 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T133939278243Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a66a71616068ae69dc70417b29c60c3ba9a362cb438599bdf873417ee17414bb +size 5628109 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T133940265692Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T133940265692Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2dd945560273816aab96a99b2e49f1254105b1cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T133940265692Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854843392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104556497408, + "active_bytes_peak_delta": 221696, + "after": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4628679680, + "inactive_split_bytes_peak": 7789264896, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:40.265291Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854843392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104556497408, + "allocated_bytes_peak_delta": 221696, + "before": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556275712, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556275712, + "device": 0, + "inactive_split_bytes_current": 4630776832, + "inactive_split_bytes_peak": 7646802944, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:39.405715Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4628679680, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7789264896, + "inactive_split_bytes_peak_delta": 142461952, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108106088448, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T133940265692Z.pickle", + "timestamp_utc": "2026-08-01T13:39:40.371713Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T133940265692Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T133940265692Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..722ebef460c68eb2aca4db7d9db8c354468428b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T133940265692Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcc37e6b2b3cca54892db19e863f8fda8d200c6d8d05f1d400a06b69693b92db +size 5617591 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T133941263669Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T133941263669Z.json new file mode 100644 index 0000000000000000000000000000000000000000..343265df759198e115e38b67ae4a17ee72b65d1b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T133941263669Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854843392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104556497408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4628679680, + "inactive_split_bytes_peak": 7789264896, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:41.263264Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854843392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104556497408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4628679680, + "inactive_split_bytes_peak": 7789264896, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:40.386610Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4628679680, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7789264896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108106088448, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T133941263669Z.pickle", + "timestamp_utc": "2026-08-01T13:39:41.367002Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T133941263669Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T133941263669Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..274fa165941f6aef69ef29705c79532d0e4808f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T133941263669Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3eabc5861d0cca6e322f34e26b5891ccd741feedc37dd7ca7c803daf74bca3d +size 5625834 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T133942863549Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T133942863549Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d5722bda5327389d6fc47d69cb273de90a074a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T133942863549Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854843392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104556497408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4857269248, + "inactive_split_bytes_peak": 7789264896, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:42.863117Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854843392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104556497408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4628679680, + "inactive_split_bytes_peak": 7789264896, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:41.381711Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4857269248, + "inactive_split_bytes_current_delta": 228589568, + "inactive_split_bytes_peak": 7789264896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108106088448, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T133942863549Z.pickle", + "timestamp_utc": "2026-08-01T13:39:42.969620Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T133942863549Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T133942863549Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b1de822f63f82aae17c0ddab0eb2f321df71f740 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T133942863549Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d720805bf73d0cb2a2df47518cf9e9cea50f6016efd20512010ee87e0a37ebcc +size 5617232 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T133943904519Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T133943904519Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0216763d9528f95dde660f10b4eadc6bb0d6fd14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T133943904519Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854843392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104556497408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4855172096, + "inactive_split_bytes_peak": 7789264896, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:43.904117Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854843392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104556497408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4857269248, + "inactive_split_bytes_peak": 7789264896, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:42.984344Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4855172096, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7789264896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108106088448, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T133943904519Z.pickle", + "timestamp_utc": "2026-08-01T13:39:44.006813Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T133943904519Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T133943904519Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..416aeee455a5689256e1947c35ce55f39dcb3233 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T133943904519Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e59671d092015ce74e31530fca928f890d3f8b31c1646229766fe25f8c4643a +size 5625384 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T133944910816Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T133944910816Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4740fbe2d9a3564539cb81a5307b3d876aca380e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T133944910816Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854843392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104556497408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4624485376, + "inactive_split_bytes_peak": 7789264896, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:44.910430Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854843392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104556497408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4855172096, + "inactive_split_bytes_peak": 7789264896, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:44.021470Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4624485376, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 7789264896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108106088448, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T133944910816Z.pickle", + "timestamp_utc": "2026-08-01T13:39:45.011848Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T133944910816Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T133944910816Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2aaeed953dec6431521667afa0a7b766c3213c2b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T133944910816Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4d6621d55ec630e5cc432068bb6c93330a16da6ee0b35494307b05b16fd57bd +size 5617035 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T133945902090Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T133945902090Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34337da134fb5b86582f673e05c336a90d0a55e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T133945902090Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854843392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104556497408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4622388224, + "inactive_split_bytes_peak": 7789264896, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:45.901695Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854843392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104556497408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4624485376, + "inactive_split_bytes_peak": 7789264896, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:45.026242Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4622388224, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7789264896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108106088448, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T133945902090Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.002230Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T133945902090Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T133945902090Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..88c1ab3aac099fa5a63f1236f351443a046fbff6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T133945902090Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bf02eaad471d9f531af675e0ebe21d1bf9e27e613ea0a544395b88668121b82 +size 5625306 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T133946896337Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T133946896337Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c7c8ca709471b9b379e5093f4c7cf73f784e0383 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T133946896337Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854843392, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104556497408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4626582528, + "inactive_split_bytes_peak": 7789264896, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:46.895928Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854843392, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104556497408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4622388224, + "inactive_split_bytes_peak": 7789264896, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:46.016615Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4626582528, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 7789264896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108106088448, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T133946896337Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.996774Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T133946896337Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T133946896337Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3342e35fb0de0ce52862ebb475fe440b0d987d89 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T133946896337Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8a022f0bffc5a9515b2135357d346f22e7a04c1d640a87535e5a8ce66cbf897 +size 5617190 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T134038691632Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T134038691632Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e462ff3f011bfec1ca7f20c0352ef21e0d7670e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T134038691632Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804200448, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104556497408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66804200448, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66804200448, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4352166912, + "inactive_split_bytes_peak": 7789264896, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:40:38.691153Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804200448, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104556497408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854843392, + "active_bytes_peak": 104556497408, + "allocated_bytes_current": 66854843392, + "allocated_bytes_peak": 104556497408, + "device": 0, + "inactive_split_bytes_current": 4626582528, + "inactive_split_bytes_peak": 7789264896, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_peak": 108106088448, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:47.011073Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4352166912, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 7789264896, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108106088448, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108106088448, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T134038691632Z.pickle", + "timestamp_utc": "2026-08-01T13:40:38.888584Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T134038691632Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T134038691632Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d13ff7d55a4db889037eb29abc4dbc1f1e04a311 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T134038691632Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f4e67a91213484dfcf8fa77ac261a9942860570bbc86d838264d2ee96a27a52 +size 10209973 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T133938277106Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T133938277106Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b2b2e91735a7f890bc017e1bbb27c162b333db73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T133938277106Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 80922977280, + "active_bytes_peak_delta": 45456641024, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80922977280, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80922977280, + "device": 1, + "inactive_split_bytes_current": 4845980672, + "inactive_split_bytes_peak": 5130607616, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 97536442368, + "reserved_bytes_peak": 97536442368, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:39:38.276529Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 80922977280, + "allocated_bytes_peak_delta": 45456641024, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:38:29.685208Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4845980672, + "inactive_split_bytes_current_delta": 4713161728, + "inactive_split_bytes_peak": 5130607616, + "inactive_split_bytes_peak_delta": 4970221568, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 97536442368, + "reserved_bytes_current_delta": 58292436992, + "reserved_bytes_peak": 97536442368, + "reserved_bytes_peak_delta": 58292436992, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T133938277106Z.pickle", + "timestamp_utc": "2026-08-01T13:39:38.387743Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T133938277106Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T133938277106Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d7cc9bad007163db35ebace357ae84429e540b9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T133938277106Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6262c110e92db6814d38d235e37188260073fc34057aa210796a7efe5145e78d +size 5386525 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T133939278203Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T133939278203Z.json new file mode 100644 index 0000000000000000000000000000000000000000..07b6b5a787a037e2fa04ac5981f636865af1db8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T133939278203Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104557400576, + "active_bytes_peak_delta": 23634423296, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104557400576, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104557400576, + "device": 1, + "inactive_split_bytes_current": 4627876864, + "inactive_split_bytes_peak": 7786686464, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:39.277788Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104557400576, + "allocated_bytes_peak_delta": 23634423296, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80922977280, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80922977280, + "device": 1, + "inactive_split_bytes_current": 4845980672, + "inactive_split_bytes_peak": 5130607616, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 97536442368, + "reserved_bytes_peak": 97536442368, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:39:38.407268Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4627876864, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 7786686464, + "inactive_split_bytes_peak_delta": 2656078848, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 10856955904, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 10856955904, + "segment_count_current": 555, + "segment_count_current_delta": 43, + "segment_count_peak": 555, + "segment_count_peak_delta": 43, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T133939278203Z.pickle", + "timestamp_utc": "2026-08-01T13:39:39.390886Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T133939278203Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T133939278203Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c7dad6fab66d7e7db739898dd9ce885aab6de469 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T133939278203Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fce33a356e38bcea1567327974f7739e5fcf8f3848407f61965be21e4d209cf +size 5622430 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T133940265544Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T133940265544Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fcdcd6f61b584b921aa7dc78c0d6b14f0bd98560 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T133940265544Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558491648, + "active_bytes_peak_delta": 1091072, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4625779712, + "inactive_split_bytes_peak": 7786686464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:40.265128Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558491648, + "allocated_bytes_peak_delta": 1091072, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104557400576, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104557400576, + "device": 1, + "inactive_split_bytes_current": 4627876864, + "inactive_split_bytes_peak": 7786686464, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:39.406366Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4625779712, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7786686464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T133940265544Z.pickle", + "timestamp_utc": "2026-08-01T13:39:40.370632Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T133940265544Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T133940265544Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..17eeeb5289c659c39965d8d1150991cc98a8b32b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T133940265544Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44037a9c2656e7235f63c683d6665e7564a3b288029fe79999c315daa62e0d93 +size 5614741 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T133941263548Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T133941263548Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ccb50cdfc2c98a2a748a3aa311f646baea5837c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T133941263548Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558491648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4860660736, + "inactive_split_bytes_peak": 7786686464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:41.263130Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558491648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4625779712, + "inactive_split_bytes_peak": 7786686464, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:40.385453Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4860660736, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 7786686464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T133941263548Z.pickle", + "timestamp_utc": "2026-08-01T13:39:41.377530Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T133941263548Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T133941263548Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c7f905cd60705428beddbd600a41d8f89f6aaf07 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T133941263548Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27caf32579d4c5f46b486a22c8905c7e0e74203ca97c3ab2925ac3d8d6c7793b +size 5616000 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T133942863621Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T133942863621Z.json new file mode 100644 index 0000000000000000000000000000000000000000..461161448de98a38e9e15932449db212e2d83eeb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T133942863621Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558491648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4619488256, + "inactive_split_bytes_peak": 7786686464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:42.863143Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558491648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4860660736, + "inactive_split_bytes_peak": 7786686464, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:41.400678Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4619488256, + "inactive_split_bytes_current_delta": -241172480, + "inactive_split_bytes_peak": 7786686464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T133942863621Z.pickle", + "timestamp_utc": "2026-08-01T13:39:42.984007Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T133942863621Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T133942863621Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ae4e190a7b98f7084716b105925877192ba517d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T133942863621Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:414b6638fc47609fc386df110c0cf8ea9028c86755b5e98880c2ac4a50b72b4e +size 5614311 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T133943904603Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T133943904603Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fbb3a4ba5661f72e1847911f2cd5cd1e8248f94a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T133943904603Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558491648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4617391104, + "inactive_split_bytes_peak": 7786686464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:43.904116Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558491648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4619488256, + "inactive_split_bytes_peak": 7786686464, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:43.026935Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4617391104, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7786686464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T133943904603Z.pickle", + "timestamp_utc": "2026-08-01T13:39:44.016604Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T133943904603Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T133943904603Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f8d29e899c111bfde5552f604b01d24d0f9847b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T133943904603Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2ffe54f7286270230d84a0559eed1ac71000ed044f3a175839fc58ff783c39a +size 5620337 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T133944910924Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T133944910924Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0b2009f12d4a647fe7f937264d8641ac51f2375a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T133944910924Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558491648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4621585408, + "inactive_split_bytes_peak": 7786686464, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:44.910430Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558491648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4617391104, + "inactive_split_bytes_peak": 7786686464, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:44.028282Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4621585408, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 7786686464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T133944910924Z.pickle", + "timestamp_utc": "2026-08-01T13:39:45.009460Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T133944910924Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T133944910924Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf00119c1bb35c425cf262769ffc7d95b5052e94 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T133944910924Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc2c8546af2e43641ab8a9d7dcc0ef1176371dc47d3f4b012ff0c7afa48b1e5d +size 5614227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T133945902181Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T133945902181Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a28d67803f3abfa014da29fc002d16bb242a5fe9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T133945902181Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558491648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4619488256, + "inactive_split_bytes_peak": 7786686464, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:45.901694Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558491648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4621585408, + "inactive_split_bytes_peak": 7786686464, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:45.021035Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4619488256, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7786686464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T133945902181Z.pickle", + "timestamp_utc": "2026-08-01T13:39:45.997628Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T133945902181Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T133945902181Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ab5c08f07f431cdc4818211ace227ec8cb911dd3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T133945902181Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90d7d486a8fa5c7b538f931e2b46e58ef1c53b4543471f6f68c3edab613b1a59 +size 5620131 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T133946896435Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T133946896435Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ce4a2c2b3f37c1615ff58cf50ce961e307107e01 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T133946896435Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558491648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4858563584, + "inactive_split_bytes_peak": 7786686464, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:46.895921Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558491648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4619488256, + "inactive_split_bytes_peak": 7786686464, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:46.008766Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4858563584, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 7786686464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T133946896435Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.992636Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T133946896435Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T133946896435Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..84aba8ef7452a90fa18462a4553bb188fe72d79f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T133946896435Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3f1e578f68175dcdba918f3c9e89768d19a929298e2cfc63baa92286d9ef497 +size 5614293 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T134038691628Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T134038691628Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7aa68cb68c5e1f0ac9a7e5f5780f8304d876970c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T134038691628Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104558491648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4349266944, + "inactive_split_bytes_peak": 7786686464, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:40:38.691159Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104558491648, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558491648, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558491648, + "device": 1, + "inactive_split_bytes_current": 4858563584, + "inactive_split_bytes_peak": 7786686464, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:47.004078Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4349266944, + "inactive_split_bytes_current_delta": -509296640, + "inactive_split_bytes_peak": 7786686464, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T134038691628Z.pickle", + "timestamp_utc": "2026-08-01T13:40:38.885063Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T134038691628Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T134038691628Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f1c626cdbfe441ec2163229209f7731513b15d66 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T134038691628Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25e761e0432f00b997081380da5963f9f27dcc0fb9c4980d2b1aee1fc118d30f +size 9648878 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T133938277272Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T133938277272Z.json new file mode 100644 index 0000000000000000000000000000000000000000..72fbaeea4210ebcfbdea17bd47776397ea3ef4dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T133938277272Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 80919658496, + "active_bytes_peak_delta": 45453322240, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80919658496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80919658496, + "device": 2, + "inactive_split_bytes_current": 4785163264, + "inactive_split_bytes_peak": 5641067520, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98050244608, + "reserved_bytes_peak": 98050244608, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:39:38.276664Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 80919658496, + "allocated_bytes_peak_delta": 45453322240, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:38:29.685303Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4785163264, + "inactive_split_bytes_current_delta": 4652344320, + "inactive_split_bytes_peak": 5641067520, + "inactive_split_bytes_peak_delta": 5480681472, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98050244608, + "reserved_bytes_current_delta": 58806239232, + "reserved_bytes_peak": 98050244608, + "reserved_bytes_peak_delta": 58806239232, + "segment_count_current": 516, + "segment_count_current_delta": 445, + "segment_count_peak": 516, + "segment_count_peak_delta": 445, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T133938277272Z.pickle", + "timestamp_utc": "2026-08-01T13:39:38.392251Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T133938277272Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T133938277272Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f43c15189149d06c0e37635d752d15eb1111be92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T133938277272Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49155a881ef6c0ac32d1b26b9643fb356b365e05f139b733d58a4f02b24ec8e0 +size 5387217 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T133939278188Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T133939278188Z.json new file mode 100644 index 0000000000000000000000000000000000000000..670e3c06b8e0fe3eff089527803e12170be56401 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T133939278188Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104557249024, + "active_bytes_peak_delta": 23637590528, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104557249024, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104557249024, + "device": 2, + "inactive_split_bytes_current": 4562865152, + "inactive_split_bytes_peak": 7986763776, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:39.277780Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104557249024, + "allocated_bytes_peak_delta": 23637590528, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80919658496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80919658496, + "device": 2, + "inactive_split_bytes_current": 4785163264, + "inactive_split_bytes_peak": 5641067520, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98050244608, + "reserved_bytes_peak": 98050244608, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:39:38.411185Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4562865152, + "inactive_split_bytes_current_delta": -222298112, + "inactive_split_bytes_peak": 7986763776, + "inactive_split_bytes_peak_delta": 2345696256, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108819120128, + "reserved_bytes_current_delta": 10768875520, + "reserved_bytes_peak": 108819120128, + "reserved_bytes_peak_delta": 10768875520, + "segment_count_current": 556, + "segment_count_current_delta": 40, + "segment_count_peak": 556, + "segment_count_peak_delta": 40, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T133939278188Z.pickle", + "timestamp_utc": "2026-08-01T13:39:39.394307Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T133939278188Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T133939278188Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..339f6fc37a11fdcd3db5c2e376f7d183cc463236 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T133939278188Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2086f578a51856361c9f6b18e5cdd8caffcb185d71f71fe0bef9ffe6bcb34c47 +size 5622953 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T133940265541Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T133940265541Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9754b879679fed85d335da244ef95d29d2dafba3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T133940265541Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558361088, + "active_bytes_peak_delta": 1112064, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4567059456, + "inactive_split_bytes_peak": 7986763776, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:40.265130Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558361088, + "allocated_bytes_peak_delta": 1112064, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104557249024, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104557249024, + "device": 2, + "inactive_split_bytes_current": 4562865152, + "inactive_split_bytes_peak": 7986763776, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:39.409532Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4567059456, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 7986763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108819120128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108819120128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T133940265541Z.pickle", + "timestamp_utc": "2026-08-01T13:39:40.372402Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T133940265541Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T133940265541Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a94f2553a3beeeadfe28ce8a40dfff90c78dc36c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T133940265541Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d67efd0bbe230ea7e5459b99b9bd50130a4d2957d520284c10e64ccfa9489651 +size 5616622 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T133941263549Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T133941263549Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a31ed37f3d69c14da717a8fdb19c9104fe2d82b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T133941263549Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558361088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4789357568, + "inactive_split_bytes_peak": 7986763776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:41.263132Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558361088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4567059456, + "inactive_split_bytes_peak": 7986763776, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:40.387420Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4789357568, + "inactive_split_bytes_current_delta": 222298112, + "inactive_split_bytes_peak": 7986763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108819120128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108819120128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T133941263549Z.pickle", + "timestamp_utc": "2026-08-01T13:39:41.369068Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T133941263549Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T133941263549Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5c85ea5f5ebb24fed8b45b1ab6121b7faa10a9d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T133941263549Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad0bcc0ec45ee42b33c4f57a8c76a36208d68b10f207f6eb3e147ecf84e48ad7 +size 5619191 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T133942863423Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T133942863423Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a994c998e65f0d989cb35649b705f9d361726726 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T133942863423Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558361088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4797746176, + "inactive_split_bytes_peak": 7986763776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:42.863007Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558361088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4789357568, + "inactive_split_bytes_peak": 7986763776, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:41.383840Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4797746176, + "inactive_split_bytes_current_delta": 8388608, + "inactive_split_bytes_peak": 7986763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108819120128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108819120128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T133942863423Z.pickle", + "timestamp_utc": "2026-08-01T13:39:42.971823Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T133942863423Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T133942863423Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b73c08f6978aeb61b10d071c01770ca7633f7865 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T133942863423Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36eb574a177e622cb4ecca302a05105550956e6d66d64fc003cee05f48bdb441 +size 5616174 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T133943904425Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T133943904425Z.json new file mode 100644 index 0000000000000000000000000000000000000000..17d8eac8126e96a0f311a2ae4b748d2b6fd1cb2d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T133943904425Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558361088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4804037632, + "inactive_split_bytes_peak": 7986763776, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:43.904018Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558361088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4797746176, + "inactive_split_bytes_peak": 7986763776, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:42.986493Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4804037632, + "inactive_split_bytes_current_delta": 6291456, + "inactive_split_bytes_peak": 7986763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108819120128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108819120128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T133943904425Z.pickle", + "timestamp_utc": "2026-08-01T13:39:44.009314Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T133943904425Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T133943904425Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4bcc45e2abf157fc633407fcfe237ba7f302af70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T133943904425Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67bb641c9bf3520863c28380a8d765eb594d23d15f7b5ca029470dbbf812dc6f +size 5619236 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T133944910839Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T133944910839Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9daac48e98410045d321ec53e13ebcc9de440fdc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T133944910839Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558361088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4560768000, + "inactive_split_bytes_peak": 7986763776, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:44.910391Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558361088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4804037632, + "inactive_split_bytes_peak": 7986763776, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:44.024003Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4560768000, + "inactive_split_bytes_current_delta": -243269632, + "inactive_split_bytes_peak": 7986763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108819120128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108819120128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T133944910839Z.pickle", + "timestamp_utc": "2026-08-01T13:39:45.013625Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T133944910839Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T133944910839Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d78022aeb5a37bfa035899365a90b64da04ef3ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T133944910839Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a465f8d6b8b5e685cc530546ec6d8fca06d1a71c566f22534b9de13f9486e850 +size 5616135 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T133945902023Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T133945902023Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b965101cbfbc1c06b069b36c52ea367176d52d4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T133945902023Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558361088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4556573696, + "inactive_split_bytes_peak": 7986763776, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:45.901619Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558361088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4560768000, + "inactive_split_bytes_peak": 7986763776, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:45.028268Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4556573696, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 7986763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108819120128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108819120128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T133945902023Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.003934Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T133945902023Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T133945902023Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd9e2c11b5828599d342342195e872283a1c1da4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T133945902023Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e59302001f2919fe04c05c8eaf413066fd709e18251a3948b1c2aed754fd6e4c +size 5618965 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T133946896320Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T133946896320Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0c039791ddd197325799bde91ab41e2b47a77993 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T133946896320Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558361088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4558670848, + "inactive_split_bytes_peak": 7986763776, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:46.895873Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558361088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4556573696, + "inactive_split_bytes_peak": 7986763776, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:46.018679Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4558670848, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 7986763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108819120128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108819120128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T133946896320Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.999208Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T133946896320Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T133946896320Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f70130b76c70d0702575fc343d657962b6642fa5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T133946896320Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a60de3669a516c684b9b8854c70da5cd663027160d0a661206cf7beec005e716 +size 5617472 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T134038691634Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T134038691634Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6df26ed0a28680a3fcd17837acfdbe20912edd8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T134038691634Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104558361088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4288449536, + "inactive_split_bytes_peak": 7986763776, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:40:38.691147Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104558361088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558361088, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558361088, + "device": 2, + "inactive_split_bytes_current": 4558670848, + "inactive_split_bytes_peak": 7986763776, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108819120128, + "reserved_bytes_peak": 108819120128, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:47.013933Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4288449536, + "inactive_split_bytes_current_delta": -270221312, + "inactive_split_bytes_peak": 7986763776, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108819120128, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108819120128, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T134038691634Z.pickle", + "timestamp_utc": "2026-08-01T13:40:38.894516Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T134038691634Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T134038691634Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e26f54db7d6427bf0fdcfe33567aa3d3b4c52105 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T134038691634Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82ce5a62e0f41741009c92f5d92d74167cbb7fb79d2de1d9ee2e10f71045681e +size 9640839 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T133938277350Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T133938277350Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a15316af04f660164435fb6b16f62b08850856ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T133938277350Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 80925206528, + "active_bytes_peak_delta": 45458870272, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80925206528, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80925206528, + "device": 3, + "inactive_split_bytes_current": 4904700928, + "inactive_split_bytes_peak": 5423684608, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 97595162624, + "reserved_bytes_peak": 97595162624, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:39:38.276715Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 80925206528, + "allocated_bytes_peak_delta": 45458870272, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:38:29.685352Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4904700928, + "inactive_split_bytes_current_delta": 4771881984, + "inactive_split_bytes_peak": 5423684608, + "inactive_split_bytes_peak_delta": 5263298560, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 97595162624, + "reserved_bytes_current_delta": 58351157248, + "reserved_bytes_peak": 97595162624, + "reserved_bytes_peak_delta": 58351157248, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T133938277350Z.pickle", + "timestamp_utc": "2026-08-01T13:39:38.393139Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T133938277350Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T133938277350Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2121e830ab7f093e4d1e0bdacd74f80c1ef4894e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T133938277350Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6b58f0aa5791d51464407a60f59fe2ca6e78fe3a0306493e14349d39b9b2304 +size 5386561 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T133939279459Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T133939279459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3698cb219ffe8a1b35b2c96ff1cf62c249494454 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T133939279459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559739904, + "active_bytes_peak_delta": 23634533376, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4686597120, + "inactive_split_bytes_peak": 7669528576, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:39.279001Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559739904, + "allocated_bytes_peak_delta": 23634533376, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80925206528, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80925206528, + "device": 3, + "inactive_split_bytes_current": 4904700928, + "inactive_split_bytes_peak": 5423684608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 97595162624, + "reserved_bytes_peak": 97595162624, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:39:38.412386Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4686597120, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 7669528576, + "inactive_split_bytes_peak_delta": 2245843968, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 10798235648, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 10798235648, + "segment_count_current": 556, + "segment_count_current_delta": 44, + "segment_count_peak": 556, + "segment_count_peak_delta": 44, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T133939279459Z.pickle", + "timestamp_utc": "2026-08-01T13:39:39.392545Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T133939279459Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T133939279459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d4c1f5efb552a9798395780766d7af23cef342e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T133939279459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c9f37eeed3d686216db9ff35591f52386258dc76d765da079a9617d891ba148 +size 5623760 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T133940265560Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T133940265560Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c57f7c688e7b264bc5ea058938daa5678b52359f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T133940265560Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559739904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4684499968, + "inactive_split_bytes_peak": 7757000704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:40.265164Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559739904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4686597120, + "inactive_split_bytes_peak": 7669528576, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:39.407891Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4684499968, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7757000704, + "inactive_split_bytes_peak_delta": 87472128, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T133940265560Z.pickle", + "timestamp_utc": "2026-08-01T13:39:40.380871Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T133940265560Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T133940265560Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..076a75ba784d3a025cf9d249dc1dc4ed5a38c6b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T133940265560Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa53087c1e1d56ad788883a7e726ba0a7595f09b7ce292b695769ed641ef8730 +size 5615541 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T133941263579Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T133941263579Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5dacf0b33b6eb12a3db6ee0f7f90922aee769b00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T133941263579Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559739904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4684499968, + "inactive_split_bytes_peak": 7757000704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:41.263167Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559739904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4684499968, + "inactive_split_bytes_peak": 7757000704, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:40.399753Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4684499968, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7757000704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T133941263579Z.pickle", + "timestamp_utc": "2026-08-01T13:39:41.366660Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T133941263579Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T133941263579Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f9167d31f1550b0839f49ff0ef617f3ee4138894 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T133941263579Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:858768ad8e6fad74b2e034312e5c55aa374c7c7852f7b50f96e459b4048dacf0 +size 5616819 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T133942863424Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T133942863424Z.json new file mode 100644 index 0000000000000000000000000000000000000000..102a35d0578e96a3ba6266a1f25fbccd149547d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T133942863424Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559739904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4913089536, + "inactive_split_bytes_peak": 7757000704, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:42.863008Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559739904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4684499968, + "inactive_split_bytes_peak": 7757000704, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:41.380632Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4913089536, + "inactive_split_bytes_current_delta": 228589568, + "inactive_split_bytes_peak": 7757000704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T133942863424Z.pickle", + "timestamp_utc": "2026-08-01T13:39:42.969180Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T133942863424Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T133942863424Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..62e603a76748fbd270185d8821b2106de71e7d87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T133942863424Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38372b977daa2d31766dc86bb60bdf5c9f62e053bd793539a6adaa5074dd63d9 +size 5615230 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T133943904438Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T133943904438Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a08abe7d151dd53088e86beab523145b83a3d42 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T133943904438Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559739904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4910992384, + "inactive_split_bytes_peak": 7757000704, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:43.904032Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559739904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4913089536, + "inactive_split_bytes_peak": 7757000704, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:42.982304Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4910992384, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7757000704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T133943904438Z.pickle", + "timestamp_utc": "2026-08-01T13:39:44.002463Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T133943904438Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T133943904438Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e3547ce293aa7e590f50eb740a7a7b595d3bff2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T133943904438Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c85a59c4d2d42a286e066c07f9bf70daf0a98068c4cc8e1c8291f94f4dfebb85 +size 5621166 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T133944910771Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T133944910771Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fcc988b3d1fe247e64420708ea105055e3776324 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T133944910771Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559739904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4680305664, + "inactive_split_bytes_peak": 7757000704, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:44.910361Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559739904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4910992384, + "inactive_split_bytes_peak": 7757000704, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:44.015252Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4680305664, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 7757000704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T133944910771Z.pickle", + "timestamp_utc": "2026-08-01T13:39:45.009242Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T133944910771Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T133944910771Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ae9a9764d03b9725af9bd99cc562ac3f05de1da4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T133944910771Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51b6174c2f454756bd9125f36aa7a6e62eabba9f4806a7833469d97b92216298 +size 5614979 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T133945902027Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T133945902027Z.json new file mode 100644 index 0000000000000000000000000000000000000000..09e32337057b430c78ba93695d37745b52702cf0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T133945902027Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559739904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4678208512, + "inactive_split_bytes_peak": 7757000704, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:45.901620Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559739904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4680305664, + "inactive_split_bytes_peak": 7757000704, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:45.021971Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4678208512, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7757000704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T133945902027Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.000907Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T133945902027Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T133945902027Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e74ff43ebe92665ed013b9afd4a2dde982c1a90 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T133945902027Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0700741adc9bc9e1f9ba64c07bd1e40d1fc79060d30d04b10c0011c621032d99 +size 5620953 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T133946896295Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T133946896295Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab0589354fca813f37caa02ce03ead8182b89bac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T133946896295Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559739904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4682402816, + "inactive_split_bytes_peak": 7757000704, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:46.895866Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559739904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4678208512, + "inactive_split_bytes_peak": 7757000704, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:46.013649Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4682402816, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 7757000704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T133946896295Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.997800Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T133946896295Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T133946896295Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..796b0891c306d7be3102eabc6c1d29f2554bff49 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T133946896295Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27cc793bf9b15c731fdab98b7a354df613fd2bdc96a0c75e667283bacfa2be24 +size 5615168 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T134038691630Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T134038691630Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1071ca162ad6a7df5765da66f5f47de132d36430 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T134038691630Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104559739904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4407987200, + "inactive_split_bytes_peak": 7757000704, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:40:38.691155Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104559739904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559739904, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559739904, + "device": 3, + "inactive_split_bytes_current": 4682402816, + "inactive_split_bytes_peak": 7757000704, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108393398272, + "reserved_bytes_peak": 108393398272, + "segment_count_current": 556, + "segment_count_peak": 556, + "timestamp_utc": "2026-08-01T13:39:47.010745Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4407987200, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 7757000704, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108393398272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108393398272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 556, + "segment_count_current_delta": 0, + "segment_count_peak": 556, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T134038691630Z.pickle", + "timestamp_utc": "2026-08-01T13:40:38.898900Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T134038691630Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T134038691630Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b2211935aca1d93ccbb7db2d46627fb0ea39c88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T134038691630Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a940b3971d495efc6d7f801678ec6e064c7451df99a16ec237170daa4759482b +size 9649667 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T133938277158Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T133938277158Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69c4e55382863d3362a35bd63c47314288968a57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T133938277158Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854817792, + "active_bytes_current_delta": 31622920192, + "active_bytes_peak": 80921077760, + "active_bytes_peak_delta": 45454299136, + "after": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 80921077760, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 80921077760, + "device": 4, + "inactive_split_bytes_current": 4788088832, + "inactive_split_bytes_peak": 4936349696, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 97756643328, + "reserved_bytes_peak": 97756643328, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:39:38.276702Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854817792, + "allocated_bytes_current_delta": 31622920192, + "allocated_bytes_peak": 80921077760, + "allocated_bytes_peak_delta": 45454299136, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:38:29.684961Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4788088832, + "inactive_split_bytes_current_delta": 4655712256, + "inactive_split_bytes_peak": 4936349696, + "inactive_split_bytes_peak_delta": 4782940160, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 97756643328, + "reserved_bytes_current_delta": 58512637952, + "reserved_bytes_peak": 97756643328, + "reserved_bytes_peak_delta": 58512637952, + "segment_count_current": 518, + "segment_count_current_delta": 447, + "segment_count_peak": 518, + "segment_count_peak_delta": 447, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T133938277158Z.pickle", + "timestamp_utc": "2026-08-01T13:39:38.389095Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T133938277158Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T133938277158Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bc9a2cbb549ead914095c876b8fa0531525962aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T133938277158Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86738900ba3b4e8f03105639bfd68129a00925660acdf59b2937ae849a693124 +size 5396030 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T133939278197Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T133939278197Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fcf29fac62c38a68d087249193a94ba437db83fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T133939278197Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854817792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104554342912, + "active_bytes_peak_delta": 23633265152, + "after": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4565790720, + "inactive_split_bytes_peak": 7861809152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:39.277753Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854817792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104554342912, + "allocated_bytes_peak_delta": 23633265152, + "before": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 80921077760, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 80921077760, + "device": 4, + "inactive_split_bytes_current": 4788088832, + "inactive_split_bytes_peak": 4936349696, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 97756643328, + "reserved_bytes_peak": 97756643328, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:39:38.408252Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4565790720, + "inactive_split_bytes_current_delta": -222298112, + "inactive_split_bytes_peak": 7861809152, + "inactive_split_bytes_peak_delta": 2925459456, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108072534016, + "reserved_bytes_current_delta": 10315890688, + "reserved_bytes_peak": 108072534016, + "reserved_bytes_peak_delta": 10315890688, + "segment_count_current": 555, + "segment_count_current_delta": 37, + "segment_count_peak": 555, + "segment_count_peak_delta": 37, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T133939278197Z.pickle", + "timestamp_utc": "2026-08-01T13:39:39.391186Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T133939278197Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T133939278197Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7353a48aa91e06750ab441169406c45c26b19c22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T133939278197Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc87d850ea0410dba42450fa1478ddfc34291ffa8ad24520a7aecf06b58c53e9 +size 5627509 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T133940265588Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T133940265588Z.json new file mode 100644 index 0000000000000000000000000000000000000000..90080cd1a81aa472f390e2f3d13b8d03383a3845 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T133940265588Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854817792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104554342912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4569985024, + "inactive_split_bytes_peak": 7861809152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:40.265178Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854817792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104554342912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4565790720, + "inactive_split_bytes_peak": 7861809152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:39.406403Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4569985024, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 7861809152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108072534016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108072534016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T133940265588Z.pickle", + "timestamp_utc": "2026-08-01T13:39:40.372219Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T133940265588Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T133940265588Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..37c64fd5d0cdd83c6cb9a7c29b8a0254de5ddf4c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T133940265588Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:515fa3ebebb5e046772ce2ccab98e0769d312ebb8e5c2737fc8b3efe543ae675 +size 5618430 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T133941263552Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T133941263552Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c4b52435b207e64fe5822d189dca54ed0a82ed60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T133941263552Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854817792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104554342912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4792283136, + "inactive_split_bytes_peak": 7861809152, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:41.263125Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854817792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104554342912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4569985024, + "inactive_split_bytes_peak": 7861809152, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:40.387031Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4792283136, + "inactive_split_bytes_current_delta": 222298112, + "inactive_split_bytes_peak": 7861809152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108072534016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108072534016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T133941263552Z.pickle", + "timestamp_utc": "2026-08-01T13:39:41.367923Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T133941263552Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T133941263552Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..81ab9a7f5b481e089015ff720cfc03b6ce49b2e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T133941263552Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fbbb08d699fb87536162b499a4f5622deda62740d815b3de8820fae059e9c28 +size 5626470 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T133942863424Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T133942863424Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65fea6ae6f0af24aa88c77a059335650ebd7a3ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T133942863424Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854817792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104554342912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4565790720, + "inactive_split_bytes_peak": 7861809152, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:42.862984Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854817792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104554342912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4792283136, + "inactive_split_bytes_peak": 7861809152, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:41.382582Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4565790720, + "inactive_split_bytes_current_delta": -226492416, + "inactive_split_bytes_peak": 7861809152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108072534016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108072534016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T133942863424Z.pickle", + "timestamp_utc": "2026-08-01T13:39:42.970243Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T133942863424Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T133942863424Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..64a0767d529076a3647fb386b2f337c4605c7219 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T133942863424Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c74efb11d20bf8ad3c77ea1320663b8dbd9a27d013e04bf9214ba0cb9590067 +size 5617978 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T133943904415Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T133943904415Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f4f3667e87f257cb3bfb84d8d0fc131d48d5cb7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T133943904415Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854817792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104554342912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4806963200, + "inactive_split_bytes_peak": 7861809152, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:43.903993Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854817792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104554342912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4565790720, + "inactive_split_bytes_peak": 7861809152, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:42.984999Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4806963200, + "inactive_split_bytes_current_delta": 241172480, + "inactive_split_bytes_peak": 7861809152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108072534016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108072534016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T133943904415Z.pickle", + "timestamp_utc": "2026-08-01T13:39:44.008410Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T133943904415Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T133943904415Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..592edd36400eae9b6227eafc35f72c131c8979a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T133943904415Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6e88108243fc8b8fadf29cdc5a764703e464ce4e74b5982266bdda21122a70f +size 5627979 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T133944910782Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T133944910782Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9636af7f9458be6aa9115cfb65dac33668321a4a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T133944910782Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854817792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104554342912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4563693568, + "inactive_split_bytes_peak": 7861809152, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:44.910365Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854817792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104554342912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4806963200, + "inactive_split_bytes_peak": 7861809152, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:44.023189Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4563693568, + "inactive_split_bytes_current_delta": -243269632, + "inactive_split_bytes_peak": 7861809152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108072534016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108072534016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T133944910782Z.pickle", + "timestamp_utc": "2026-08-01T13:39:45.013167Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T133944910782Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T133944910782Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11120911f5011c2a0ff21e06f9aeceb4f1085326 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T133944910782Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bca8e3dc60c46f3f6417565186622f98a75807dddcf22fd492ccc4d385d19bd3 +size 5626113 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T133945902002Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T133945902002Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b936d0c428acdbc5fd151268ac8f340989eeb8b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T133945902002Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854817792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104554342912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4559499264, + "inactive_split_bytes_peak": 7861809152, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:45.901592Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854817792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104554342912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4563693568, + "inactive_split_bytes_peak": 7861809152, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:45.027950Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4559499264, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 7861809152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108072534016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108072534016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T133945902002Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.004809Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T133945902002Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T133945902002Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c46790760161725deed35302c9d99065eb6e73ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T133945902002Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c001a1b326a7d18bff775267613681a2fd5aac43e1ad2b12c4e7aee113277e20 +size 5627742 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T133946896219Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T133946896219Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e8412765d317084563a06b302f60179c26e8d91b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T133946896219Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854817792, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104554342912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4561596416, + "inactive_split_bytes_peak": 7861809152, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:46.895811Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854817792, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104554342912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4559499264, + "inactive_split_bytes_peak": 7861809152, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:46.019436Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4561596416, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 7861809152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108072534016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108072534016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T133946896219Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.999775Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T133946896219Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T133946896219Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c7a44acd1977a28ab00c3243203e4dae85610bd6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T133946896219Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45df6b1c1d3a15e945a4cc98c03fd606d3f198d71e7392d87372835794082006 +size 5626096 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T134038691642Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T134038691642Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bec40468471700eab68a91fb10f54db83b849ae1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T134038691642Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804174848, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104554342912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66804174848, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66804174848, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4291375104, + "inactive_split_bytes_peak": 7861809152, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:40:38.691179Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804174848, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104554342912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854817792, + "active_bytes_peak": 104554342912, + "allocated_bytes_current": 66854817792, + "allocated_bytes_peak": 104554342912, + "device": 4, + "inactive_split_bytes_current": 4561596416, + "inactive_split_bytes_peak": 7861809152, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 108072534016, + "reserved_bytes_peak": 108072534016, + "segment_count_current": 555, + "segment_count_peak": 555, + "timestamp_utc": "2026-08-01T13:39:47.014327Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4291375104, + "inactive_split_bytes_current_delta": -270221312, + "inactive_split_bytes_peak": 7861809152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 108072534016, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108072534016, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 555, + "segment_count_current_delta": 0, + "segment_count_peak": 555, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T134038691642Z.pickle", + "timestamp_utc": "2026-08-01T13:40:38.899439Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T134038691642Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T134038691642Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a538a142742dce7f75c32e4dfdd60bc630fc88e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T134038691642Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bd09a496a936fe6cb0b1a2763acad75f646db689636e63a84cd2d2974e1cba1 +size 10211536 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T133938277191Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T133938277191Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f293baf207343521fa937cb5ca0a97a125aeafe9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T133938277191Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 80920749056, + "active_bytes_peak_delta": 45454412800, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80920749056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80920749056, + "device": 5, + "inactive_split_bytes_current": 4827106304, + "inactive_split_bytes_peak": 5334273536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 98532589568, + "reserved_bytes_peak": 98532589568, + "segment_count_current": 521, + "segment_count_peak": 521, + "timestamp_utc": "2026-08-01T13:39:38.276595Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 80920749056, + "allocated_bytes_peak_delta": 45454412800, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:38:29.685275Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4827106304, + "inactive_split_bytes_current_delta": 4694287360, + "inactive_split_bytes_peak": 5334273536, + "inactive_split_bytes_peak_delta": 5173887488, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 98532589568, + "reserved_bytes_current_delta": 59288584192, + "reserved_bytes_peak": 98532589568, + "reserved_bytes_peak_delta": 59288584192, + "segment_count_current": 521, + "segment_count_current_delta": 450, + "segment_count_peak": 521, + "segment_count_peak_delta": 450, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T133938277191Z.pickle", + "timestamp_utc": "2026-08-01T13:39:38.387439Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T133938277191Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T133938277191Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e58aaf28bac0ffc97e73b1e1a89d487bc69e8f48 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T133938277191Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d8df36526da42092c37ae9b79afcd82fed1ec2d35ac6ca42edd81191228ae5d +size 5388048 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T133939278210Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T133939278210Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c79bb4b8445e823eef191716f51c744c7a608ed7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T133939278210Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104558973440, + "active_bytes_peak_delta": 23638224384, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558973440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558973440, + "device": 5, + "inactive_split_bytes_current": 4604808192, + "inactive_split_bytes_peak": 7535990784, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:39.277797Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104558973440, + "allocated_bytes_peak_delta": 23638224384, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80920749056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80920749056, + "device": 5, + "inactive_split_bytes_current": 4827106304, + "inactive_split_bytes_peak": 5334273536, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 98532589568, + "reserved_bytes_peak": 98532589568, + "segment_count_current": 521, + "segment_count_peak": 521, + "timestamp_utc": "2026-08-01T13:39:38.404699Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4604808192, + "inactive_split_bytes_current_delta": -222298112, + "inactive_split_bytes_peak": 7535990784, + "inactive_split_bytes_peak_delta": 2201717248, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108538101760, + "reserved_bytes_current_delta": 10005512192, + "reserved_bytes_peak": 108538101760, + "reserved_bytes_peak_delta": 10005512192, + "segment_count_current": 557, + "segment_count_current_delta": 36, + "segment_count_peak": 557, + "segment_count_peak_delta": 36, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T133939278210Z.pickle", + "timestamp_utc": "2026-08-01T13:39:39.389147Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T133939278210Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T133939278210Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a66a87f51faee2a34504eaaa227e8ba25d8b86d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T133939278210Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03fa0dd8ca9472a289a93c0d41f447f07dce19bed9c77bd3a05718b293254e9f +size 5623283 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T133940265560Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T133940265560Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1bb9fb3b067c44978c25a9695519795f66f402e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T133940265560Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559475712, + "active_bytes_peak_delta": 502272, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4609002496, + "inactive_split_bytes_peak": 8013705216, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:40.265149Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559475712, + "allocated_bytes_peak_delta": 502272, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104558973440, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104558973440, + "device": 5, + "inactive_split_bytes_current": 4604808192, + "inactive_split_bytes_peak": 7535990784, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:39.402906Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4609002496, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8013705216, + "inactive_split_bytes_peak_delta": 477714432, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108538101760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108538101760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T133940265560Z.pickle", + "timestamp_utc": "2026-08-01T13:39:40.380186Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T133940265560Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T133940265560Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b52339d0d900dd9363b958268d9926fad37a0e63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T133940265560Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3854cd24c0f67e235d491edc2df7777a3d683aae940ff26d394944bc1de29896 +size 5617130 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T133941263533Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T133941263533Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b0efb98f451d6e74a8b9c72ab35e63b5f7cd6b77 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T133941263533Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559475712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4596419584, + "inactive_split_bytes_peak": 8013705216, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:41.263138Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559475712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4609002496, + "inactive_split_bytes_peak": 8013705216, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:40.397571Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4596419584, + "inactive_split_bytes_current_delta": -12582912, + "inactive_split_bytes_peak": 8013705216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108538101760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108538101760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T133941263533Z.pickle", + "timestamp_utc": "2026-08-01T13:39:41.362086Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T133941263533Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T133941263533Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9866855357ccbe27dc94b960c2c287d25878258a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T133941263533Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48956162cff9e50c608b2d9510a12ef70127a72ac61c4ffb4f113e5d75c529a4 +size 5619620 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T133942863421Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T133942863421Z.json new file mode 100644 index 0000000000000000000000000000000000000000..16b82b08000bfa52c9e2ad4a29222907c7e790ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T133942863421Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559475712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4839689216, + "inactive_split_bytes_peak": 8013705216, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:42.862999Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559475712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4596419584, + "inactive_split_bytes_peak": 8013705216, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:41.374994Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4839689216, + "inactive_split_bytes_current_delta": 243269632, + "inactive_split_bytes_peak": 8013705216, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108538101760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108538101760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T133942863421Z.pickle", + "timestamp_utc": "2026-08-01T13:39:42.968355Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T133942863421Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T133942863421Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93dc2542086975473f4521f04502d360e9ec8485 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T133942863421Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dd8867376c21216975933060f6cd4db3c202107f92ef01e23e9a69668a78706 +size 5616459 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T133943904426Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T133943904426Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d575f2d49cd19d8136a83dc42ba6d21ac3c2065 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T133943904426Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559475712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4845980672, + "inactive_split_bytes_peak": 8015802368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:43.904034Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559475712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4839689216, + "inactive_split_bytes_peak": 8013705216, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:42.981829Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4845980672, + "inactive_split_bytes_current_delta": 6291456, + "inactive_split_bytes_peak": 8015802368, + "inactive_split_bytes_peak_delta": 2097152, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108538101760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108538101760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T133943904426Z.pickle", + "timestamp_utc": "2026-08-01T13:39:44.002667Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T133943904426Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T133943904426Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a6cd8fa1c4f602129f596466e09cb3463a06c75a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T133943904426Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30581dc3e5fdc5a1dfd21d9ffc32778b99174f31e40d792da7895f3ff79a7a16 +size 5619442 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T133944910736Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T133944910736Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4abec9469c32aa920522f2d082cf1e6bb56e64b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T133944910736Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559475712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4602711040, + "inactive_split_bytes_peak": 8015802368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:44.910343Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559475712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4845980672, + "inactive_split_bytes_peak": 8015802368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:44.015336Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4602711040, + "inactive_split_bytes_current_delta": -243269632, + "inactive_split_bytes_peak": 8015802368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108538101760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108538101760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T133944910736Z.pickle", + "timestamp_utc": "2026-08-01T13:39:45.008964Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T133944910736Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T133944910736Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..072fdc1d064c8f61fc66feb453848703fa098006 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T133944910736Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40cd06943fe57b598313e7df8674406a31eed4a090b16939db3601acefe25c6c +size 5617764 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T133945902020Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T133945902020Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c479f3e15069be29209a13003be0a15d4adbac55 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T133945902020Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559475712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4833397760, + "inactive_split_bytes_peak": 8015802368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:45.901616Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559475712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4602711040, + "inactive_split_bytes_peak": 8015802368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:45.021657Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4833397760, + "inactive_split_bytes_current_delta": 230686720, + "inactive_split_bytes_peak": 8015802368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108538101760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108538101760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T133945902020Z.pickle", + "timestamp_utc": "2026-08-01T13:39:45.998416Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T133945902020Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T133945902020Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d7d301836041ba76155dcf85ef58c2d311a92465 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T133945902020Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bbb1e6a42bd9bef0b3673fddf34aa7e4f6db40899de716b60216df3d8a2d5c +size 5622250 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T133946896241Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T133946896241Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15af808dd935c22b4db461009d951259d061ead8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T133946896241Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104559475712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4600613888, + "inactive_split_bytes_peak": 8015802368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:46.895838Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104559475712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4833397760, + "inactive_split_bytes_peak": 8015802368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:46.010928Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4600613888, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 8015802368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108538101760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108538101760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T133946896241Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.994521Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T133946896241Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T133946896241Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..53892c1f7e958048f70d29bcee0874bc76cf162c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T133946896241Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3b6ccc5dc5cb8ae7461a2fa7ecd3b1cc39bed1c7a39e0c15e6766c8533e8749 +size 5617412 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T134038691631Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T134038691631Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6effce67609bde18ce1ef098b7b9faf74e85edef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T134038691631Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104559475712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4330392576, + "inactive_split_bytes_peak": 8015802368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:40:38.691155Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104559475712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104559475712, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104559475712, + "device": 5, + "inactive_split_bytes_current": 4600613888, + "inactive_split_bytes_peak": 8015802368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108538101760, + "reserved_bytes_peak": 108538101760, + "segment_count_current": 557, + "segment_count_peak": 557, + "timestamp_utc": "2026-08-01T13:39:47.007394Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4330392576, + "inactive_split_bytes_current_delta": -270221312, + "inactive_split_bytes_peak": 8015802368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108538101760, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108538101760, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 557, + "segment_count_current_delta": 0, + "segment_count_peak": 557, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T134038691631Z.pickle", + "timestamp_utc": "2026-08-01T13:40:38.894195Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T134038691631Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T134038691631Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..acf0adecd80e2c5775af47118e887b880efbade7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T134038691631Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc72d38a247b7f54090bd06670bc6aeab3ee0850021ed8295c54b1ad46b07406 +size 9463723 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T133938277088Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T133938277088Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c78ebc0c96455fe60a50024ffc1035726cc90262 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T133938277088Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 80920749056, + "active_bytes_peak_delta": 45454412800, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80920749056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80920749056, + "device": 6, + "inactive_split_bytes_current": 4827106304, + "inactive_split_bytes_peak": 5491851264, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98062827520, + "reserved_bytes_peak": 98062827520, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:39:38.276511Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 80920749056, + "allocated_bytes_peak_delta": 45454412800, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:38:29.685284Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4827106304, + "inactive_split_bytes_current_delta": 4694287360, + "inactive_split_bytes_peak": 5491851264, + "inactive_split_bytes_peak_delta": 5331465216, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98062827520, + "reserved_bytes_current_delta": 58818822144, + "reserved_bytes_peak": 98062827520, + "reserved_bytes_peak_delta": 58818822144, + "segment_count_current": 519, + "segment_count_current_delta": 448, + "segment_count_peak": 519, + "segment_count_peak_delta": 448, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T133938277088Z.pickle", + "timestamp_utc": "2026-08-01T13:39:38.385543Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T133938277088Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T133938277088Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..75abdf65168d36edec6cde79b493dcf58a155b3e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T133938277088Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c38948b4e4821391b601c84b437f7196aae07a9ca5aa67afa5abbe5b54e4e199 +size 5387697 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T133939278194Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T133939278194Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b9cef5e857bcf767efff5bffcb791311957ad583 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T133939278194Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104560546304, + "active_bytes_peak_delta": 23639797248, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4604808192, + "inactive_split_bytes_peak": 8030808064, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:39.277782Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104560546304, + "allocated_bytes_peak_delta": 23639797248, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 80920749056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 80920749056, + "device": 6, + "inactive_split_bytes_current": 4827106304, + "inactive_split_bytes_peak": 5491851264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98062827520, + "reserved_bytes_peak": 98062827520, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:39:38.402675Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4604808192, + "inactive_split_bytes_current_delta": -222298112, + "inactive_split_bytes_peak": 8030808064, + "inactive_split_bytes_peak_delta": 2538956800, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108349358080, + "reserved_bytes_current_delta": 10286530560, + "reserved_bytes_peak": 108349358080, + "reserved_bytes_peak_delta": 10286530560, + "segment_count_current": 554, + "segment_count_current_delta": 35, + "segment_count_peak": 554, + "segment_count_peak_delta": 35, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T133939278194Z.pickle", + "timestamp_utc": "2026-08-01T13:39:39.389302Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T133939278194Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T133939278194Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f368a7363b145b9bef6ddff2a7435401f30c28f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T133939278194Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02e619173c3340c815e5aa9a872d849b7caa1037db65540f62eb06effca25c5e +size 5623576 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T133940265546Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T133940265546Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b6261eea3bf974ddbfeb254556f7ced1395bc900 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T133940265546Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104560546304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4843883520, + "inactive_split_bytes_peak": 8030808064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:40.265134Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104560546304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4604808192, + "inactive_split_bytes_peak": 8030808064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:39.402930Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4843883520, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 8030808064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108349358080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108349358080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T133940265546Z.pickle", + "timestamp_utc": "2026-08-01T13:39:40.367696Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T133940265546Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T133940265546Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..62fcf4052db0817b239c0932d7b35219ff1a2160 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T133940265546Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2b99715de86c63c69a185f3e19d170019fdb5c5325c3fa3583dec685d79613f +size 5617730 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T133941263559Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T133941263559Z.json new file mode 100644 index 0000000000000000000000000000000000000000..43b4468ee4cda66e2b1f820c86cddaf6b68e2b01 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T133941263559Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104560546304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4596419584, + "inactive_split_bytes_peak": 8030808064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:41.263150Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104560546304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4843883520, + "inactive_split_bytes_peak": 8030808064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:40.380691Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4596419584, + "inactive_split_bytes_current_delta": -247463936, + "inactive_split_bytes_peak": 8030808064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108349358080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108349358080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T133941263559Z.pickle", + "timestamp_utc": "2026-08-01T13:39:41.363734Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T133941263559Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T133941263559Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c33375815eca80c621d55fe0d2d2f1e25d87301e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T133941263559Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43420a04aeeff7675189f0cf07d0a73ab7c48dab555da31e6ebdf2978249b0f5 +size 5620186 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T133942863406Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T133942863406Z.json new file mode 100644 index 0000000000000000000000000000000000000000..84025b3997b452c26ab3427d359e45d1a819b12e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T133942863406Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104560546304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4839689216, + "inactive_split_bytes_peak": 8030808064, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:42.862997Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104560546304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4596419584, + "inactive_split_bytes_peak": 8030808064, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:41.376745Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4839689216, + "inactive_split_bytes_current_delta": 243269632, + "inactive_split_bytes_peak": 8030808064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108349358080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108349358080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T133942863406Z.pickle", + "timestamp_utc": "2026-08-01T13:39:42.969507Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T133942863406Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T133942863406Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b494b517a9242effc691ed400b3cf801a7235164 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T133942863406Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3859ddec2fd92e47a5508a2be20f1ff74a16c98756219a2782e0b643016c00cc +size 5617387 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T133943904424Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T133943904424Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bcc79823ca0505aa65810e496369db535bc81ebd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T133943904424Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104560546304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4845980672, + "inactive_split_bytes_peak": 8030808064, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:43.904015Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104560546304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4839689216, + "inactive_split_bytes_peak": 8030808064, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:42.986001Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4845980672, + "inactive_split_bytes_current_delta": 6291456, + "inactive_split_bytes_peak": 8030808064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108349358080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108349358080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T133943904424Z.pickle", + "timestamp_utc": "2026-08-01T13:39:44.003678Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T133943904424Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T133943904424Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ecae61c364b5b136c2ae99de7a74f9797402e5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T133943904424Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d079bf67cacef02055e1684af5ccef9bd03298e51e42c1480c129a8fe98df19a +size 5620439 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T133944910772Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T133944910772Z.json new file mode 100644 index 0000000000000000000000000000000000000000..903e740f09ad876e36f15b54de8f033d522df799 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T133944910772Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104560546304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4837592064, + "inactive_split_bytes_peak": 8030808064, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:44.910350Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104560546304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4845980672, + "inactive_split_bytes_peak": 8030808064, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:44.016409Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4837592064, + "inactive_split_bytes_current_delta": -8388608, + "inactive_split_bytes_peak": 8030808064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108349358080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108349358080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T133944910772Z.pickle", + "timestamp_utc": "2026-08-01T13:39:45.010498Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T133944910772Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T133944910772Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..59f2a55cf088ad33eef41dc5145e4e8a671621bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T133944910772Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d218cb7cf05530774affd73f1dc7ca86cced4b988b0cdbceb9d3fb4198a1a0e +size 5617402 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T133945901997Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T133945901997Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1dbd6ee12fb03a23f0dfeb10ee6950f2e60a420c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T133945901997Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104560546304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4598516736, + "inactive_split_bytes_peak": 8030808064, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:45.901597Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104560546304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4837592064, + "inactive_split_bytes_peak": 8030808064, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:45.023125Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4598516736, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 8030808064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108349358080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108349358080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T133945901997Z.pickle", + "timestamp_utc": "2026-08-01T13:39:45.998706Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T133945901997Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T133945901997Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f1de608c3f0d1296c7789228b5a8e79e60bd2691 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T133945901997Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:374b6b66fc4e6899db241e0fd17f16175f6c9e9cb76fb2f3bab4afa280885293 +size 5620167 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T133946896211Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T133946896211Z.json new file mode 100644 index 0000000000000000000000000000000000000000..819ba71df1330d99cfe68f3012a993b92c0a1ab2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T133946896211Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104560546304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4835494912, + "inactive_split_bytes_peak": 8030808064, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:46.895808Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104560546304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4598516736, + "inactive_split_bytes_peak": 8030808064, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:46.011191Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4835494912, + "inactive_split_bytes_current_delta": 236978176, + "inactive_split_bytes_peak": 8030808064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108349358080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108349358080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T133946896211Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.993210Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T133946896211Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T133946896211Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4803b23a0dbe6fc0763c108b93465659ca5b9966 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T133946896211Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c98dc7973f6e2d793170cc341fbcea8204ed8f53d24a390006357dabf1c21ce +size 5618718 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T134038691614Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T134038691614Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e3825ded294dee57bafbfd8a909422659316e2c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T134038691614Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104560546304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4330392576, + "inactive_split_bytes_peak": 8030808064, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:40:38.691157Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104560546304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 104560546304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 104560546304, + "device": 6, + "inactive_split_bytes_current": 4835494912, + "inactive_split_bytes_peak": 8030808064, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108349358080, + "reserved_bytes_peak": 108349358080, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:47.005925Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4330392576, + "inactive_split_bytes_current_delta": -505102336, + "inactive_split_bytes_peak": 8030808064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108349358080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108349358080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T134038691614Z.pickle", + "timestamp_utc": "2026-08-01T13:40:38.889115Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T134038691614Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T134038691614Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b349d3a75023c0af517a83e37240d8387e797260 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T134038691614Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fb01c27f2d45b8768a92f53c31437d79c33b3546995975750cd1c5687525027 +size 9464391 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T133938277025Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T133938277025Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d6c35d0c2b0e442b791eb1c068e273d654b8e644 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T133938277025Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 31622094336, + "active_bytes_peak": 80921466880, + "active_bytes_peak_delta": 45455130624, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 80921466880, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 80921466880, + "device": 7, + "inactive_split_bytes_current": 4904700416, + "inactive_split_bytes_peak": 6564070400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 97595162624, + "reserved_bytes_peak": 97595162624, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:39:38.276537Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 31622094336, + "allocated_bytes_peak": 80921466880, + "allocated_bytes_peak_delta": 45455130624, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:38:29.685228Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4904700416, + "inactive_split_bytes_current_delta": 4771881472, + "inactive_split_bytes_peak": 6564070400, + "inactive_split_bytes_peak_delta": 6403684352, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 97595162624, + "reserved_bytes_current_delta": 58351157248, + "reserved_bytes_peak": 97595162624, + "reserved_bytes_peak_delta": 58351157248, + "segment_count_current": 512, + "segment_count_current_delta": 441, + "segment_count_peak": 512, + "segment_count_peak_delta": 441, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T133938277025Z.pickle", + "timestamp_utc": "2026-08-01T13:39:38.390746Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T133938277025Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T133938277025Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..35cb20f0bc804fa132e15048de45342b675fb719 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T133938277025Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c68ca88ce1e4d403686895e0ac80c76672533470170915b13ad2e5e24e70569c +size 5388178 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T133939278249Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T133939278249Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6784c139df0774f9c21b8211264232c72872e753 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T133939278249Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104557315072, + "active_bytes_peak_delta": 23635848192, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4686596608, + "inactive_split_bytes_peak": 7845492224, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:39.277820Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104557315072, + "allocated_bytes_peak_delta": 23635848192, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 80921466880, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 80921466880, + "device": 7, + "inactive_split_bytes_current": 4904700416, + "inactive_split_bytes_peak": 6564070400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 97595162624, + "reserved_bytes_peak": 97595162624, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:39:38.410269Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4686596608, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 7845492224, + "inactive_split_bytes_peak_delta": 1281421824, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108410175488, + "reserved_bytes_current_delta": 10815012864, + "reserved_bytes_peak": 108410175488, + "reserved_bytes_peak_delta": 10815012864, + "segment_count_current": 554, + "segment_count_current_delta": 42, + "segment_count_peak": 554, + "segment_count_peak_delta": 42, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T133939278249Z.pickle", + "timestamp_utc": "2026-08-01T13:39:39.391227Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T133939278249Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T133939278249Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ea9221895d63110906ff652e663d5b373e57deb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T133939278249Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab30265671cb00bf4f19a7e14919b501d41fa8d687b7b2164899398e8343cd1f +size 5625325 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T133940265558Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T133940265558Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d33dddc11351f0a83b8d54830d8f27f76ccac28d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T133940265558Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104557315072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4684499456, + "inactive_split_bytes_peak": 7927936512, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:40.265146Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104557315072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4686596608, + "inactive_split_bytes_peak": 7845492224, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:39.406612Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4684499456, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7927936512, + "inactive_split_bytes_peak_delta": 82444288, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108410175488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108410175488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T133940265558Z.pickle", + "timestamp_utc": "2026-08-01T13:39:40.371154Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T133940265558Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T133940265558Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f9854d7f6674c6ebccf62a12d3107eaf5691dea5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T133940265558Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4b02d317912f72f2c6c2c9213b58e81c81f6c7b893c0cb153c3ae83c59f2145 +size 5617355 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T133941263564Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T133941263564Z.json new file mode 100644 index 0000000000000000000000000000000000000000..29d7fac55f6eb0db694b29951b4b801a222e9a5c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T133941263564Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104557315072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4919380480, + "inactive_split_bytes_peak": 7927936512, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:41.263164Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104557315072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4684499456, + "inactive_split_bytes_peak": 7927936512, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:40.385963Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4919380480, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 7927936512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108410175488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108410175488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T133941263564Z.pickle", + "timestamp_utc": "2026-08-01T13:39:41.367299Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T133941263564Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T133941263564Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..edf9a611678bd45eec928b9521bc463a18259768 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T133941263564Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad86327e37c871c5b5a0557f8513073418dbc257b7e3324031ae5b5165d2b90b +size 5618562 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T133942863459Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T133942863459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..73a3ad3e977b814f100303d815f97d0328e49508 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T133942863459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104557315072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4913089024, + "inactive_split_bytes_peak": 7927936512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:42.863034Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104557315072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4919380480, + "inactive_split_bytes_peak": 7927936512, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:41.381183Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4913089024, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 7927936512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108410175488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108410175488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T133942863459Z.pickle", + "timestamp_utc": "2026-08-01T13:39:42.968862Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T133942863459Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T133942863459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1dc0efb9468bb7322e16548a7560d576d664b8e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T133942863459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:034be32d21c62dfeb440c1f3d972d931eb2021533748378228d2d6473bc3ac85 +size 5616890 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T133943904482Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T133943904482Z.json new file mode 100644 index 0000000000000000000000000000000000000000..26f281c160600f4e4147cdecb93c13123a0405b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T133943904482Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104557315072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4910991872, + "inactive_split_bytes_peak": 7927936512, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:43.904075Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104557315072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4913089024, + "inactive_split_bytes_peak": 7927936512, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:42.982178Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4910991872, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7927936512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108410175488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108410175488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T133943904482Z.pickle", + "timestamp_utc": "2026-08-01T13:39:44.005724Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T133943904482Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T133943904482Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4cd0d0a03540c9644f37db45a0d64251ced362aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T133943904482Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06622c38be48afb0f43df45e5f0513900cb515dfe0155eeb51ce0b540ca145d6 +size 5622982 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T133944910781Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T133944910781Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7b8971d9a1c1946cd5bf6b467541a6a72e54b694 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T133944910781Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104557315072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4680305152, + "inactive_split_bytes_peak": 7927936512, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:44.910387Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104557315072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4910991872, + "inactive_split_bytes_peak": 7927936512, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:44.018759Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4680305152, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 7927936512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108410175488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108410175488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T133944910781Z.pickle", + "timestamp_utc": "2026-08-01T13:39:45.016486Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T133944910781Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T133944910781Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2b0fc5c2b3ba878384fc110726248cffa194d3e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T133944910781Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0b6f5e190d2db8ea3ff1948fb5754a825912d4f6a9b474939a24f73eb3d8eb3 +size 5616787 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T133945902037Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T133945902037Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65e83afa019011b571719c4bf48b76ab5ab948c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T133945902037Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104557315072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4678208000, + "inactive_split_bytes_peak": 7927936512, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:45.901642Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104557315072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4680305152, + "inactive_split_bytes_peak": 7927936512, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:45.031970Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4678208000, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7927936512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108410175488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108410175488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T133945902037Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.001817Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T133945902037Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T133945902037Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13b5914130628dfa3595c0fab6d2709bf09f7dd9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T133945902037Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2f1faa755f05cc4b03dd4c437aca3c00ac80fd80940ded1d084965937c7e089 +size 5622659 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T133946896269Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T133946896269Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f72acb40e6dc81a4dcfe84c5599cb982edb0098 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T133946896269Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104557315072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4682402304, + "inactive_split_bytes_peak": 7927936512, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:46.895861Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104557315072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4678208000, + "inactive_split_bytes_peak": 7927936512, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:46.014641Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4682402304, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 7927936512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108410175488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108410175488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T133946896269Z.pickle", + "timestamp_utc": "2026-08-01T13:39:46.996611Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T133946896269Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T133946896269Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4f7882bea3f09d4ab3a912936cdab01430268f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T133946896269Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5aab09c138df6dece27681c75481e1c39144d4540b24ee4f3727c6d97caad6a +size 5616919 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T134038691614Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T134038691614Z.json new file mode 100644 index 0000000000000000000000000000000000000000..864029c102183b02ece1e09a71be6c98e838a330 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T134038691614Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906624, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 104557315072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906624, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66802906624, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4407986688, + "inactive_split_bytes_peak": 7927936512, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:40:38.691159Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906624, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 104557315072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 104557315072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 104557315072, + "device": 7, + "inactive_split_bytes_current": 4682402304, + "inactive_split_bytes_peak": 7927936512, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108410175488, + "reserved_bytes_peak": 108410175488, + "segment_count_current": 554, + "segment_count_peak": 554, + "timestamp_utc": "2026-08-01T13:39:47.009462Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4407986688, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 7927936512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108410175488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108410175488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 554, + "segment_count_current_delta": 0, + "segment_count_peak": 554, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T134038691614Z.pickle", + "timestamp_utc": "2026-08-01T13:40:38.887541Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T134038691614Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T134038691614Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..162778866d53e08b5f11b302a15c65ea829104e7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T134038691614Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15bc857d90923ea331b5ee20e9a3027b740324beeb544476790308ba5ac122b6 +size 9641600 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..ca3222ee2b1fa7779d881f05d28d0c0932367645 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 2048 +capacity: 1.25 +expert_dispatch_overlap: on +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6031 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 220 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T134323149891Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T134323149891Z.json new file mode 100644 index 0000000000000000000000000000000000000000..047c9b16dce60fb3d31095fa0d5289bca5b0256f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T134323149891Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441345024, + "active_bytes_current_delta": 37209889792, + "active_bytes_peak": 81168366592, + "active_bytes_peak_delta": 45702030336, + "after": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 81168366592, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 80834454528, + "device": 0, + "inactive_split_bytes_current": 5159570432, + "inactive_split_bytes_peak": 8959274496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98417246208, + "reserved_bytes_peak": 98417246208, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T13:43:23.149269Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441345024, + "allocated_bytes_current_delta": 37209889792, + "allocated_bytes_peak": 80834454528, + "allocated_bytes_peak_delta": 45368118272, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:42:15.516545Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5159570432, + "inactive_split_bytes_current_delta": 5024654336, + "inactive_split_bytes_peak": 8959274496, + "inactive_split_bytes_peak_delta": 8798642688, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98417246208, + "reserved_bytes_current_delta": 59171143680, + "reserved_bytes_peak": 98417246208, + "reserved_bytes_peak_delta": 59171143680, + "segment_count_current": 467, + "segment_count_current_delta": 395, + "segment_count_peak": 467, + "segment_count_peak_delta": 395, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T134323149891Z.pickle", + "timestamp_utc": "2026-08-01T13:43:23.267572Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T134323149891Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T134323149891Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..36e7d632222efe5b872b251d79f088f066a30b47 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T134323149891Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d5f056c6b37abc23bca8492d0598e48e3dd953015f1f07fdc7bc68428cec17 +size 5952074 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T134324309221Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T134324309221Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4bc80a53d99fb7f4b654140964d45138e40c6095 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T134324309221Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441345024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104812590080, + "active_bytes_peak_delta": 23644223488, + "after": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472309760, + "device": 0, + "inactive_split_bytes_current": 5392354304, + "inactive_split_bytes_peak": 11703218176, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:24.308694Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441345024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472309760, + "allocated_bytes_peak_delta": 23637855232, + "before": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 81168366592, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 80834454528, + "device": 0, + "inactive_split_bytes_current": 5159570432, + "inactive_split_bytes_peak": 8959274496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98417246208, + "reserved_bytes_peak": 98417246208, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T13:43:23.284526Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5392354304, + "inactive_split_bytes_current_delta": 232783872, + "inactive_split_bytes_peak": 11703218176, + "inactive_split_bytes_peak_delta": 2743943680, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_current_delta": 11580473344, + "reserved_bytes_peak": 109997719552, + "reserved_bytes_peak_delta": 11580473344, + "segment_count_current": 520, + "segment_count_current_delta": 53, + "segment_count_peak": 520, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T134324309221Z.pickle", + "timestamp_utc": "2026-08-01T13:43:24.434355Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T134324309221Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T134324309221Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a6f6f90eca6c9c10653478968518d1697b8fdd2c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T134324309221Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87ca80d0ce62570d4117da452a3809dbd1818decd112cb47ef2002a000318b17 +size 6114093 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T134325474693Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T134325474693Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c7912fe4632719ae71f4292708250150c8a8f9c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T134325474693Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441345024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104812590080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472309760, + "device": 0, + "inactive_split_bytes_current": 5279108096, + "inactive_split_bytes_peak": 11703218176, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:25.474200Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441345024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472309760, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472309760, + "device": 0, + "inactive_split_bytes_current": 5392354304, + "inactive_split_bytes_peak": 11703218176, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:24.453128Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5279108096, + "inactive_split_bytes_current_delta": -113246208, + "inactive_split_bytes_peak": 11703218176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109997719552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T134325474693Z.pickle", + "timestamp_utc": "2026-08-01T13:43:25.593632Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T134325474693Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T134325474693Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d6a4ad3bb0df2e37d4d55f7e2e7175087448ff5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T134325474693Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25bcb3195f73b58d6a375cce3bec4c27f8522edb11f9ceb3d058b5fb47149fe4 +size 6629962 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T134326899777Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T134326899777Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0633666c911d4a3f59f00f7dad6cb4b92fabf6d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T134326899777Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441345024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104812590080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472309760, + "device": 0, + "inactive_split_bytes_current": 5306371072, + "inactive_split_bytes_peak": 11703218176, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:26.899284Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441345024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472309760, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472309760, + "device": 0, + "inactive_split_bytes_current": 5279108096, + "inactive_split_bytes_peak": 11703218176, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:25.614675Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5306371072, + "inactive_split_bytes_current_delta": 27262976, + "inactive_split_bytes_peak": 11703218176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109997719552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T134326899777Z.pickle", + "timestamp_utc": "2026-08-01T13:43:27.017169Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T134326899777Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T134326899777Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5f64c250808d326f6306b8da60bdeb2b721d4a7f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T134326899777Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f31c3d357ef9c51a151d10c412333c7a9d44e7d8172bd156973f8a03c2c7b51b +size 6628777 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T134328353033Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T134328353033Z.json new file mode 100644 index 0000000000000000000000000000000000000000..202a162ce52817caab078ad80d83ffc4417c173d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T134328353033Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441345024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104812590080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472750080, + "device": 0, + "inactive_split_bytes_current": 5398645760, + "inactive_split_bytes_peak": 11703218176, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:28.352530Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441345024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472750080, + "allocated_bytes_peak_delta": 440320, + "before": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472309760, + "device": 0, + "inactive_split_bytes_current": 5306371072, + "inactive_split_bytes_peak": 11703218176, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:27.035285Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5398645760, + "inactive_split_bytes_current_delta": 92274688, + "inactive_split_bytes_peak": 11703218176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109997719552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T134328353033Z.pickle", + "timestamp_utc": "2026-08-01T13:43:28.469233Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T134328353033Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T134328353033Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..487344087db89b49e65d5568e78126c9244e4294 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T134328353033Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97b1d7889e0129d34d686eca5a940a3d20a89a9ccda384c805905d58b3d96343 +size 6628429 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T134329515395Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T134329515395Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8a60e3b482f65f365a90cbd5b8902e26bfd55a40 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T134329515395Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441345024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104812590080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472750080, + "device": 0, + "inactive_split_bytes_current": 5075684352, + "inactive_split_bytes_peak": 11703218176, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:29.514894Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441345024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472750080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472750080, + "device": 0, + "inactive_split_bytes_current": 5398645760, + "inactive_split_bytes_peak": 11703218176, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:28.487498Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5075684352, + "inactive_split_bytes_current_delta": -322961408, + "inactive_split_bytes_peak": 11703218176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109997719552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T134329515395Z.pickle", + "timestamp_utc": "2026-08-01T13:43:29.635385Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T134329515395Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T134329515395Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de6276e695a33666151f46d00534258959555be4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T134329515395Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dc33673c28d91a0ea3cbf17e9b78cc8477fcf78b09d94c1c6b7e7007b3a6c24 +size 6646155 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T134330695134Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T134330695134Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bd58d08cdfba9a5e485cf6f9cb46c52fca8b520a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T134330695134Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441345024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104812590080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472750080, + "device": 0, + "inactive_split_bytes_current": 5425908736, + "inactive_split_bytes_peak": 11703218176, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:30.694596Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441345024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472750080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472750080, + "device": 0, + "inactive_split_bytes_current": 5075684352, + "inactive_split_bytes_peak": 11703218176, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:29.654037Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5425908736, + "inactive_split_bytes_current_delta": 350224384, + "inactive_split_bytes_peak": 11703218176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109997719552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T134330695134Z.pickle", + "timestamp_utc": "2026-08-01T13:43:30.809777Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T134330695134Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T134330695134Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76bf828db87fcd03c696d817c415105c70157103 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T134330695134Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de54034cf63d4236c0b32ff8770ad159859f510437f7b2ea5826c45c49935c1c +size 6627711 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T134331997037Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T134331997037Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a3f8ed83f90e006fbded6a2b9105d6156ba6f1e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T134331997037Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441345024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104812590080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472750080, + "device": 0, + "inactive_split_bytes_current": 5428005888, + "inactive_split_bytes_peak": 11703218176, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:31.996511Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441345024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472750080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472750080, + "device": 0, + "inactive_split_bytes_current": 5425908736, + "inactive_split_bytes_peak": 11703218176, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:30.828072Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5428005888, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11703218176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109997719552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T134331997037Z.pickle", + "timestamp_utc": "2026-08-01T13:43:32.110329Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T134331997037Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T134331997037Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8e778159174f2a87e9ccf6aff15ed31d89c247e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T134331997037Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4491030d8107dd9943b7ebe5f5cdd1a52ea7e3419305347b7563fb1a294e13e +size 6627691 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T134333161428Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T134333161428Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18baede09dec0134a0073dcf98f8b7e8769b1db7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T134333161428Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441345024, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104812590080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472750080, + "device": 0, + "inactive_split_bytes_current": 4983409664, + "inactive_split_bytes_peak": 11703218176, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:33.160878Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441345024, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472750080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472750080, + "device": 0, + "inactive_split_bytes_current": 5428005888, + "inactive_split_bytes_peak": 11703218176, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:32.128508Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4983409664, + "inactive_split_bytes_current_delta": -444596224, + "inactive_split_bytes_peak": 11703218176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109997719552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T134333161428Z.pickle", + "timestamp_utc": "2026-08-01T13:43:33.273515Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T134333161428Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T134333161428Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c6e0a139d451b9b67ab3e20f5f5478d6c8ed18e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T134333161428Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:895363dec00e9b2a629c385cdac74194b7821c3d0d5c434e1e0671a56a225b12 +size 6627476 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T134426036510Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T134426036510Z.json new file mode 100644 index 0000000000000000000000000000000000000000..39e5c1d54c4d8f4a620d68d62180a3a2a99de48f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T134426036510Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61167055872, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 104812590080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61167055872, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 61167055872, + "allocated_bytes_peak": 104472750080, + "device": 0, + "inactive_split_bytes_current": 6742920192, + "inactive_split_bytes_peak": 11703218176, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:44:26.036046Z", + "world_size": 8 + }, + "allocated_bytes_current": 61167055872, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104472750080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441345024, + "active_bytes_peak": 104812590080, + "allocated_bytes_current": 72441345024, + "allocated_bytes_peak": 104472750080, + "device": 0, + "inactive_split_bytes_current": 4983409664, + "inactive_split_bytes_peak": 11703218176, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_peak": 109997719552, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:33.291319Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6742920192, + "inactive_split_bytes_current_delta": 1759510528, + "inactive_split_bytes_peak": 11703218176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 109997719552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109997719552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T134426036510Z.pickle", + "timestamp_utc": "2026-08-01T13:44:26.254257Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T134426036510Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T134426036510Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a41dc03dbb704c6fdaeed944ded02501a4b4d3b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T134426036510Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51bb45324150ad223183a36a84ee1dd73fc5c1a4c86b04941e0ed7151b69aedd +size 10541858 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T134323149700Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T134323149700Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d32df595afa1d6937e0b2761fc17ecca3785c87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T134323149700Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 81521040384, + "active_bytes_peak_delta": 46054704128, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 81521040384, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80828497920, + "device": 1, + "inactive_split_bytes_current": 4892429312, + "inactive_split_bytes_peak": 8691719680, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 97943289856, + "reserved_bytes_peak": 97943289856, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T13:43:23.149123Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 80828497920, + "allocated_bytes_peak_delta": 45362161664, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:42:15.516823Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4892429312, + "inactive_split_bytes_current_delta": 4759610368, + "inactive_split_bytes_peak": 8691719680, + "inactive_split_bytes_peak_delta": 8531333632, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 97943289856, + "reserved_bytes_current_delta": 58699284480, + "reserved_bytes_peak": 97943289856, + "reserved_bytes_peak_delta": 58699284480, + "segment_count_current": 463, + "segment_count_current_delta": 392, + "segment_count_peak": 463, + "segment_count_peak_delta": 392, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T134323149700Z.pickle", + "timestamp_utc": "2026-08-01T13:43:23.270293Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T134323149700Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T134323149700Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df726bf2d58097b7bea5f7eb4b0d07e5f70c157e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T134323149700Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dde37166f2c23eff4d75df97637eca50aa816a81a9cfdaaa0592d171a010e12b +size 5942756 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T134324309075Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T134324309075Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e306d8385bf2a125320797c0895a7d4022fdd4ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T134324309075Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104817319936, + "active_bytes_peak_delta": 23296279552, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104817319936, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5066492928, + "inactive_split_bytes_peak": 11984799744, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 110635253760, + "reserved_bytes_peak": 110635253760, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:24.308657Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469759488, + "allocated_bytes_peak_delta": 23641261568, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 81521040384, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80828497920, + "device": 1, + "inactive_split_bytes_current": 4892429312, + "inactive_split_bytes_peak": 8691719680, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 97943289856, + "reserved_bytes_peak": 97943289856, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T13:43:23.289047Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5066492928, + "inactive_split_bytes_current_delta": 174063616, + "inactive_split_bytes_peak": 11984799744, + "inactive_split_bytes_peak_delta": 3293080064, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 110635253760, + "reserved_bytes_current_delta": 12691963904, + "reserved_bytes_peak": 110635253760, + "reserved_bytes_peak_delta": 12691963904, + "segment_count_current": 519, + "segment_count_current_delta": 56, + "segment_count_peak": 519, + "segment_count_peak_delta": 56, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T134324309075Z.pickle", + "timestamp_utc": "2026-08-01T13:43:24.426588Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T134324309075Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T134324309075Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c22ced725b07633932bb257d182b46d18a8a8e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T134324309075Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4a35cbc0ad1fe2c4431cfe30a2ea75b9beb8c09d47b9d98dfa703a643e95f73 +size 6626789 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T134325474461Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T134325474461Z.json new file mode 100644 index 0000000000000000000000000000000000000000..df63a5556901dd88e873b34a08c2ac96a2833b15 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T134325474461Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104817319936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104817319936, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5244750848, + "inactive_split_bytes_peak": 11984799744, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:25.474060Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469759488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104817319936, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5066492928, + "inactive_split_bytes_peak": 11984799744, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 110635253760, + "reserved_bytes_peak": 110635253760, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:24.443139Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5244750848, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 11984799744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 110637350912, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 110637350912, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 520, + "segment_count_current_delta": 1, + "segment_count_peak": 520, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T134325474461Z.pickle", + "timestamp_utc": "2026-08-01T13:43:25.593689Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T134325474461Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T134325474461Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e2165e1ea671c94d132e92b3f109b03d8444ac0c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T134325474461Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c52a5c003a92f9dacbc03827dbf24686af2819ebad4ae8de42f8bebc091230a0 +size 6628974 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T134326899535Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T134326899535Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e14aba9ba98b5431cbb7f9d0f76378f9e5ed81ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T134326899535Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104817319936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104817319936, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5186030592, + "inactive_split_bytes_peak": 11984799744, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:26.899113Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469759488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104817319936, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5244750848, + "inactive_split_bytes_peak": 11984799744, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:25.612232Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5186030592, + "inactive_split_bytes_current_delta": -58720256, + "inactive_split_bytes_peak": 11984799744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 110637350912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110637350912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T134326899535Z.pickle", + "timestamp_utc": "2026-08-01T13:43:27.009272Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T134326899535Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T134326899535Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..89173d1ebc71e962df2469e4023c878985461a6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T134326899535Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29d802fd2384bc9e0107d8e8007f51f4999537b1bc20891864474a485cec7b72 +size 6627519 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T134328352828Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T134328352828Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5313012fc8461ba1df7a7a57458141ffc96555d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T134328352828Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104818430976, + "active_bytes_peak_delta": 1111040, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104818430976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5540449280, + "inactive_split_bytes_peak": 11984799744, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:28.352408Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469759488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104817319936, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5186030592, + "inactive_split_bytes_peak": 11984799744, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:27.025200Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5540449280, + "inactive_split_bytes_current_delta": 354418688, + "inactive_split_bytes_peak": 11984799744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 110637350912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110637350912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T134328352828Z.pickle", + "timestamp_utc": "2026-08-01T13:43:28.462542Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T134328352828Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T134328352828Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..87d87e9bee4ea437a39372942577b9f4abf29a9e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T134328352828Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a10702ce331c45332228f10672462e2d7925b5b6f56ebaf68b37b47abe6c07d +size 6627441 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T134329515142Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T134329515142Z.json new file mode 100644 index 0000000000000000000000000000000000000000..50eace3cabe48a1f14bca5def893de043ab6b820 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T134329515142Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104818430976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104818430976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5188127744, + "inactive_split_bytes_peak": 11984799744, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:29.514736Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469759488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104818430976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5540449280, + "inactive_split_bytes_peak": 11984799744, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:28.478196Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5188127744, + "inactive_split_bytes_current_delta": -352321536, + "inactive_split_bytes_peak": 11984799744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 110637350912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110637350912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T134329515142Z.pickle", + "timestamp_utc": "2026-08-01T13:43:29.623557Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T134329515142Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T134329515142Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..723d1eaf943316689d5d48078c2a825fb588a5c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T134329515142Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1420f6b36b33e5e067bf08d61b9dcca92610c3caec19f19bbd73d54f4e50f0d +size 6627048 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T134330694886Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T134330694886Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7fe72e79bf10c935552122d580b1f22734a7082d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T134330694886Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104818430976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104818430976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5420911616, + "inactive_split_bytes_peak": 11984799744, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:30.694453Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469759488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104818430976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5188127744, + "inactive_split_bytes_peak": 11984799744, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:29.639076Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5420911616, + "inactive_split_bytes_current_delta": 232783872, + "inactive_split_bytes_peak": 11984799744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 110637350912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110637350912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T134330694886Z.pickle", + "timestamp_utc": "2026-08-01T13:43:30.803338Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T134330694886Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T134330694886Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2a46e3fe881e4e9d3ebd64a196ceb350e8ad0abb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T134330694886Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a36bf12c1d2958c1c5afab41a37c8d8ca4c544d3bd21af38794e6266967b2cff +size 6626794 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T134331996802Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T134331996802Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d08077aee39fcfe53e364c4cc083453eba95c7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T134331996802Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105160491520, + "active_bytes_peak_delta": 342060544, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105160491520, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5041327104, + "inactive_split_bytes_peak": 11984799744, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:31.996369Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469759488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104818430976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5420911616, + "inactive_split_bytes_peak": 11984799744, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:30.819032Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5041327104, + "inactive_split_bytes_current_delta": -379584512, + "inactive_split_bytes_peak": 11984799744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 110637350912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110637350912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T134331996802Z.pickle", + "timestamp_utc": "2026-08-01T13:43:32.103980Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T134331996802Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T134331996802Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..edbc8b1635a5eb57a9186a9499fb726c799e9015 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T134331996802Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cdfd846d3c78a0be4d48eac4a31ca1e2a857971b7df9a5f7071ffefcf1a7454 +size 6626542 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T134333161120Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T134333161120Z.json new file mode 100644 index 0000000000000000000000000000000000000000..803bf723c60798edabfa8eedfd61cbc263c05df8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T134333161120Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105160491520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105160491520, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5097950208, + "inactive_split_bytes_peak": 11984799744, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:33.160726Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469759488, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105160491520, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5041327104, + "inactive_split_bytes_peak": 11984799744, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:32.119421Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5097950208, + "inactive_split_bytes_current_delta": 56623104, + "inactive_split_bytes_peak": 11984799744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 110637350912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110637350912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T134333161120Z.pickle", + "timestamp_utc": "2026-08-01T13:43:33.271684Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T134333161120Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T134333161120Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f621c050599a38cad356a51ff2cfb05eeaa27cd7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T134333161120Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98f76558f57b91fc73e5219c1b931c8f03442ae5e30f8e3b1fa23d95c635a7e8 +size 6625985 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T134426036569Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T134426036569Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2740530074d1428f147b1a9f41f1c33263a923a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T134426036569Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105160491520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 105160491520, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 104471791616, + "device": 1, + "inactive_split_bytes_current": 6591122432, + "inactive_split_bytes_peak": 11984799744, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:44:26.036071Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104471791616, + "allocated_bytes_peak_delta": 2032128, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105160491520, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469759488, + "device": 1, + "inactive_split_bytes_current": 5097950208, + "inactive_split_bytes_peak": 11984799744, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 110637350912, + "reserved_bytes_peak": 110637350912, + "segment_count_current": 520, + "segment_count_peak": 520, + "timestamp_utc": "2026-08-01T13:43:33.291207Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6591122432, + "inactive_split_bytes_current_delta": 1493172224, + "inactive_split_bytes_peak": 11984799744, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 110637350912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110637350912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 520, + "segment_count_current_delta": 0, + "segment_count_peak": 520, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T134426036569Z.pickle", + "timestamp_utc": "2026-08-01T13:44:26.240567Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T134426036569Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T134426036569Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..07526b7334cb67da8838fbbe2396de51d2b89e9e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T134426036569Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:356fd4cae10ed17cd5932575cdf3033bfe649abdc8203662be8e3330a7363831 +size 9722857 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T134323149709Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T134323149709Z.json new file mode 100644 index 0000000000000000000000000000000000000000..72112c783b9c20bcf5116939b615ffc63e6fbf52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T134323149709Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 81521040384, + "active_bytes_peak_delta": 46054704128, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 81521040384, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80828497920, + "device": 2, + "inactive_split_bytes_current": 5244750848, + "inactive_split_bytes_peak": 8809160192, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98413051904, + "reserved_bytes_peak": 98413051904, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T13:43:23.149108Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 80828497920, + "allocated_bytes_peak_delta": 45362161664, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:42:15.516871Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5244750848, + "inactive_split_bytes_current_delta": 5111931904, + "inactive_split_bytes_peak": 8809160192, + "inactive_split_bytes_peak_delta": 8648774144, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98413051904, + "reserved_bytes_current_delta": 59169046528, + "reserved_bytes_peak": 98413051904, + "reserved_bytes_peak_delta": 59169046528, + "segment_count_current": 466, + "segment_count_current_delta": 395, + "segment_count_peak": 466, + "segment_count_peak_delta": 395, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T134323149709Z.pickle", + "timestamp_utc": "2026-08-01T13:43:23.269625Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T134323149709Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T134323149709Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c19e9d059ef0f89e92a1afc66daf078c971f1e01 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T134323149709Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aea1ba5f02727cfc7ebb92c54b8fd19651b046f71af626406c6f34ae1f49910 +size 5943257 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T134324309011Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T134324309011Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ba984e2c62c98229e22940bb15de10c87a57d376 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T134324309011Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105151679488, + "active_bytes_peak_delta": 23630639104, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105151679488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104470414848, + "device": 2, + "inactive_split_bytes_current": 5303471104, + "inactive_split_bytes_peak": 11660658688, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109678952448, + "reserved_bytes_peak": 109678952448, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:24.308594Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104470414848, + "allocated_bytes_peak_delta": 23641916928, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 81521040384, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80828497920, + "device": 2, + "inactive_split_bytes_current": 5244750848, + "inactive_split_bytes_peak": 8809160192, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98413051904, + "reserved_bytes_peak": 98413051904, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T13:43:23.287880Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5303471104, + "inactive_split_bytes_current_delta": 58720256, + "inactive_split_bytes_peak": 11660658688, + "inactive_split_bytes_peak_delta": 2851498496, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109678952448, + "reserved_bytes_current_delta": 11265900544, + "reserved_bytes_peak": 109678952448, + "reserved_bytes_peak_delta": 11265900544, + "segment_count_current": 517, + "segment_count_current_delta": 51, + "segment_count_peak": 517, + "segment_count_peak_delta": 51, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T134324309011Z.pickle", + "timestamp_utc": "2026-08-01T13:43:24.424018Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T134324309011Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T134324309011Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..97f153279761d64d7cdc83d1f7861f57c807f4fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T134324309011Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cdee5daba10fe7ce6b3af9d0ce501b17939f1080693d3050a3adb86e7fc21b6 +size 6614831 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T134325474459Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T134325474459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a1cb2dabd17667044ac1bf841c4e182d4fab4dd2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T134325474459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105158218752, + "active_bytes_peak_delta": 6539264, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105158218752, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104470414848, + "device": 2, + "inactive_split_bytes_current": 5655792640, + "inactive_split_bytes_peak": 11660658688, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:25.474050Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104470414848, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105151679488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104470414848, + "device": 2, + "inactive_split_bytes_current": 5303471104, + "inactive_split_bytes_peak": 11660658688, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109678952448, + "reserved_bytes_peak": 109678952448, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:24.438181Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5655792640, + "inactive_split_bytes_current_delta": 352321536, + "inactive_split_bytes_peak": 11660658688, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109681049600, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 109681049600, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 518, + "segment_count_current_delta": 1, + "segment_count_peak": 518, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T134325474459Z.pickle", + "timestamp_utc": "2026-08-01T13:43:25.579953Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T134325474459Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T134325474459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d83fe900611cf2dadb293350afb45cdfcb1fb988 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T134325474459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bd85c6db28053490ca34dd779b273f8613e408d57ecfb03bf626a85563b13c4 +size 6628104 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T134326899564Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T134326899564Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0679ef9e0d03a1d82430f9fa3532c51487457e01 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T134326899564Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105158218752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105158218752, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472028672, + "device": 2, + "inactive_split_bytes_current": 5330734080, + "inactive_split_bytes_peak": 11660658688, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:26.899147Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472028672, + "allocated_bytes_peak_delta": 1613824, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105158218752, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104470414848, + "device": 2, + "inactive_split_bytes_current": 5655792640, + "inactive_split_bytes_peak": 11660658688, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:25.593746Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5330734080, + "inactive_split_bytes_current_delta": -325058560, + "inactive_split_bytes_peak": 11660658688, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109681049600, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109681049600, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T134326899564Z.pickle", + "timestamp_utc": "2026-08-01T13:43:27.006298Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T134326899564Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T134326899564Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dd10b56f8f95d7a148564cfdee31d0d774a51f36 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T134326899564Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f2d5bd56695b48cc60ddbe9f47fc54380b469c578f5a561a933bc2b3c3de9a4 +size 6627204 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T134328352802Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T134328352802Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ab008e32c84cf9b04eb8759ad5a713a69c9ea9c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T134328352802Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159721984, + "active_bytes_peak_delta": 1503232, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159721984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472028672, + "device": 2, + "inactive_split_bytes_current": 4980509696, + "inactive_split_bytes_peak": 11660658688, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:28.352374Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472028672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105158218752, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472028672, + "device": 2, + "inactive_split_bytes_current": 5330734080, + "inactive_split_bytes_peak": 11660658688, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:27.019993Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4980509696, + "inactive_split_bytes_current_delta": -350224384, + "inactive_split_bytes_peak": 11660658688, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109681049600, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109681049600, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T134328352802Z.pickle", + "timestamp_utc": "2026-08-01T13:43:28.459555Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T134328352802Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T134328352802Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..724ac56e5124cf80410a6b000187fd60bd9f4d95 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T134328352802Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d04bfa40962f68a47ceac5d784b867008e07bada16cefcc06c94334c81fd5b28 +size 6623562 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T134329515134Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T134329515134Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d1f5cb0f26afe8b7092f7cc722a4a6a28333dfc0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T134329515134Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159721984, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159721984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472028672, + "device": 2, + "inactive_split_bytes_current": 5393648640, + "inactive_split_bytes_peak": 11660658688, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:29.514719Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472028672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159721984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472028672, + "device": 2, + "inactive_split_bytes_current": 4980509696, + "inactive_split_bytes_peak": 11660658688, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:28.472970Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5393648640, + "inactive_split_bytes_current_delta": 413138944, + "inactive_split_bytes_peak": 11660658688, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109681049600, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109681049600, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T134329515134Z.pickle", + "timestamp_utc": "2026-08-01T13:43:29.620442Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T134329515134Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T134329515134Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2703423ca1194b71ed3e09585b34c6a0cb9fab5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T134329515134Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aea67b765b0726c3426770ba90a0f5c9934b9985f78972feb89133cc5fd04e07 +size 6626479 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T134330694840Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T134330694840Z.json new file mode 100644 index 0000000000000000000000000000000000000000..13738fa7144cbec92326f847c23670f3f9fe1e8d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T134330694840Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159721984, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159721984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472028672, + "device": 2, + "inactive_split_bytes_current": 5160864768, + "inactive_split_bytes_peak": 11660658688, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:30.694422Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472028672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159721984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472028672, + "device": 2, + "inactive_split_bytes_current": 5393648640, + "inactive_split_bytes_peak": 11660658688, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:29.634026Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5160864768, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 11660658688, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109681049600, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109681049600, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T134330694840Z.pickle", + "timestamp_utc": "2026-08-01T13:43:30.798183Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T134330694840Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T134330694840Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..883b259cda3be6f63fc009b97ef2c5d99bac1f1c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T134330694840Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b10d24545fcb0a996d3e638043ba0ae401203ec3475afff35acaebdb5045d711 +size 6623005 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T134331996762Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T134331996762Z.json new file mode 100644 index 0000000000000000000000000000000000000000..353b13edcc06be2ddcf08371d9f5c1dd6fb07e11 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T134331996762Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159867904, + "active_bytes_peak_delta": 145920, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159867904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472028672, + "device": 2, + "inactive_split_bytes_current": 5775330304, + "inactive_split_bytes_peak": 11660658688, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:31.996356Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472028672, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159721984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472028672, + "device": 2, + "inactive_split_bytes_current": 5160864768, + "inactive_split_bytes_peak": 11660658688, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:30.811434Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5775330304, + "inactive_split_bytes_current_delta": 614465536, + "inactive_split_bytes_peak": 11660658688, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109681049600, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109681049600, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T134331996762Z.pickle", + "timestamp_utc": "2026-08-01T13:43:32.108450Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T134331996762Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T134331996762Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e5e773f88c934e4872040ab9869cd9c748aeeea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T134331996762Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df0d91bab609625ebfb2d95434f390e1d9d31bf13e23b09245752cc9f4b6cec4 +size 6623014 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T134333161114Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T134333161114Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db3f5463ebb62ed0d754e11105c71ef40beb21f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T134333161114Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159867904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159867904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476288512, + "device": 2, + "inactive_split_bytes_current": 5420911616, + "inactive_split_bytes_peak": 11660658688, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:33.160704Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476288512, + "allocated_bytes_peak_delta": 4259840, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159867904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472028672, + "device": 2, + "inactive_split_bytes_current": 5775330304, + "inactive_split_bytes_peak": 11660658688, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:32.124491Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5420911616, + "inactive_split_bytes_current_delta": -354418688, + "inactive_split_bytes_peak": 11660658688, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109681049600, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109681049600, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T134333161114Z.pickle", + "timestamp_utc": "2026-08-01T13:43:33.266461Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T134333161114Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T134333161114Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..374b58068a1e39cd123da1f15c464e5b0378019b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T134333161114Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:226f2613e223a2cc3a30dc69744aad3b7d0733538293dadb6ad0729f7e817aed +size 6626371 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T134426036548Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T134426036548Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eaf99a750bfa870d038b78687009ff19b8fdc3a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T134426036548Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105159867904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 105159867904, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 104476288512, + "device": 2, + "inactive_split_bytes_current": 6003919872, + "inactive_split_bytes_peak": 11660658688, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:44:26.036055Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104476288512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159867904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104476288512, + "device": 2, + "inactive_split_bytes_current": 5420911616, + "inactive_split_bytes_peak": 11660658688, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 109681049600, + "reserved_bytes_peak": 109681049600, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:33.279871Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6003919872, + "inactive_split_bytes_current_delta": 583008256, + "inactive_split_bytes_peak": 11660658688, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 109681049600, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109681049600, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T134426036548Z.pickle", + "timestamp_utc": "2026-08-01T13:44:26.236951Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T134426036548Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T134426036548Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b13b33081d601bf66fb467e83612f86d27d7f60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T134426036548Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66750c391cb4d83247263f35ab86fe91db2fbe4b807966956cf8ea4463b02321 +size 9723228 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T134323149783Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T134323149783Z.json new file mode 100644 index 0000000000000000000000000000000000000000..19578bb39eb45f6dcaa8c8fec8ca18b8ab99424a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T134323149783Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 81174640640, + "active_bytes_peak_delta": 45708304384, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 81174640640, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80835009536, + "device": 3, + "inactive_split_bytes_current": 4951149568, + "inactive_split_bytes_peak": 8909889024, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 97309949952, + "reserved_bytes_peak": 97309949952, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T13:43:23.149173Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 80835009536, + "allocated_bytes_peak_delta": 45368673280, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:42:15.517096Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4951149568, + "inactive_split_bytes_current_delta": 4818330624, + "inactive_split_bytes_peak": 8909889024, + "inactive_split_bytes_peak_delta": 8749502976, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 97309949952, + "reserved_bytes_current_delta": 58065944576, + "reserved_bytes_peak": 97309949952, + "reserved_bytes_peak_delta": 58065944576, + "segment_count_current": 461, + "segment_count_current_delta": 390, + "segment_count_peak": 461, + "segment_count_peak_delta": 390, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T134323149783Z.pickle", + "timestamp_utc": "2026-08-01T13:43:23.267147Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T134323149783Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T134323149783Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0dfb3e8deb20e13323c60472108cec21e22942ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T134323149783Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:695b0c725429606b22a12d0dbce84d6b24f179c70affa0e3d5b87ec9f513e262 +size 5942428 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T134324309128Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T134324309128Z.json new file mode 100644 index 0000000000000000000000000000000000000000..491629a150cbac1e3483d6ce33b7b0fb7006e178 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T134324309128Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159871488, + "active_bytes_peak_delta": 23985230848, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104468318208, + "device": 3, + "inactive_split_bytes_current": 5951491072, + "inactive_split_bytes_peak": 10935174144, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108858966016, + "reserved_bytes_peak": 108858966016, + "segment_count_current": 514, + "segment_count_peak": 514, + "timestamp_utc": "2026-08-01T13:43:24.308625Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104468318208, + "allocated_bytes_peak_delta": 23633308672, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 81174640640, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80835009536, + "device": 3, + "inactive_split_bytes_current": 4951149568, + "inactive_split_bytes_peak": 8909889024, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 97309949952, + "reserved_bytes_peak": 97309949952, + "segment_count_current": 461, + "segment_count_peak": 461, + "timestamp_utc": "2026-08-01T13:43:23.284629Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5951491072, + "inactive_split_bytes_current_delta": 1000341504, + "inactive_split_bytes_peak": 10935174144, + "inactive_split_bytes_peak_delta": 2025285120, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108858966016, + "reserved_bytes_current_delta": 11549016064, + "reserved_bytes_peak": 108858966016, + "reserved_bytes_peak_delta": 11549016064, + "segment_count_current": 514, + "segment_count_current_delta": 53, + "segment_count_peak": 514, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T134324309128Z.pickle", + "timestamp_utc": "2026-08-01T13:43:24.428713Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T134324309128Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T134324309128Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cafb45d4a7dccd3ba10c1b6fe1aaad1e4640588a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T134324309128Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5c430c75be4072058ac08b0bb4950652737a614770e5f1236ff5f198b7caa26 +size 6082869 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T134325474507Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T134325474507Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6649602d09f10a08d5ac920824284a65513be4b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T134325474507Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159871488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469800960, + "device": 3, + "inactive_split_bytes_current": 5802593280, + "inactive_split_bytes_peak": 10935174144, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:25.474077Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104469800960, + "allocated_bytes_peak_delta": 1482752, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104468318208, + "device": 3, + "inactive_split_bytes_current": 5951491072, + "inactive_split_bytes_peak": 10935174144, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108858966016, + "reserved_bytes_peak": 108858966016, + "segment_count_current": 514, + "segment_count_peak": 514, + "timestamp_utc": "2026-08-01T13:43:24.445444Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5802593280, + "inactive_split_bytes_current_delta": -148897792, + "inactive_split_bytes_peak": 10935174144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110037565440, + "reserved_bytes_current_delta": 1178599424, + "reserved_bytes_peak": 110037565440, + "reserved_bytes_peak_delta": 1178599424, + "segment_count_current": 518, + "segment_count_current_delta": 4, + "segment_count_peak": 518, + "segment_count_peak_delta": 4, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T134325474507Z.pickle", + "timestamp_utc": "2026-08-01T13:43:25.584516Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T134325474507Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T134325474507Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4770025927affb893ba584bcd807703fa51fab7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T134325474507Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81d43fb4a513021f2b51ebbeda13b700c963d7607d36c6c26d3be023211152b0 +size 6104969 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T134326899571Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T134326899571Z.json new file mode 100644 index 0000000000000000000000000000000000000000..45f200b1c2cee9546fae0323b23a465e15476ae6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T134326899571Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159871488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472131072, + "device": 3, + "inactive_split_bytes_current": 5955685376, + "inactive_split_bytes_peak": 10935174144, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:26.899144Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472131072, + "allocated_bytes_peak_delta": 2330112, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104469800960, + "device": 3, + "inactive_split_bytes_current": 5802593280, + "inactive_split_bytes_peak": 10935174144, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:25.600964Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5955685376, + "inactive_split_bytes_current_delta": 153092096, + "inactive_split_bytes_peak": 10935174144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110037565440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110037565440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T134326899571Z.pickle", + "timestamp_utc": "2026-08-01T13:43:27.021356Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T134326899571Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T134326899571Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b8c3548b70f137be42675da25b46d9ca065cfb88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T134326899571Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:693c64644bb2d34fd2b404ac1833ba6f9d3df78ec622d21fefd7932443aed97e +size 6623682 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T134328352806Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T134328352806Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5cce0d8f14273288f1800c775659748f9c588145 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T134328352806Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159871488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472131072, + "device": 3, + "inactive_split_bytes_current": 5840342016, + "inactive_split_bytes_peak": 10935174144, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:28.352403Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472131072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472131072, + "device": 3, + "inactive_split_bytes_current": 5955685376, + "inactive_split_bytes_peak": 10935174144, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:27.040354Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5840342016, + "inactive_split_bytes_current_delta": -115343360, + "inactive_split_bytes_peak": 10935174144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110037565440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110037565440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T134328352806Z.pickle", + "timestamp_utc": "2026-08-01T13:43:28.462550Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T134328352806Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T134328352806Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..34fed4c962f74aca0d1c10f4c0a28737c04bb89e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T134328352806Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03939eb85cda96dbdc793011e75a880385deb2498d6467e820ebfa94ea653a9a +size 6626329 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T134329515164Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T134329515164Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3aa528e300f0fa1dd23a320f50deb18096f579f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T134329515164Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159871488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472131072, + "device": 3, + "inactive_split_bytes_current": 5779524608, + "inactive_split_bytes_peak": 11043164160, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:29.514754Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472131072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472131072, + "device": 3, + "inactive_split_bytes_current": 5840342016, + "inactive_split_bytes_peak": 10935174144, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:28.478440Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5779524608, + "inactive_split_bytes_current_delta": -60817408, + "inactive_split_bytes_peak": 11043164160, + "inactive_split_bytes_peak_delta": 107990016, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110037565440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110037565440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T134329515164Z.pickle", + "timestamp_utc": "2026-08-01T13:43:29.622822Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T134329515164Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T134329515164Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2494539a13e11b6da42e2c5cd51d59021070b4d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T134329515164Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eca06c5ad74e94e19bea583718b2b952ef3877ece3d809539fc11ad47e1b1bd +size 6626220 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T134330694880Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T134330694880Z.json new file mode 100644 index 0000000000000000000000000000000000000000..afb83e557bf56e6f7d1f6d015b4f525bca40caea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T134330694880Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159871488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472131072, + "device": 3, + "inactive_split_bytes_current": 5662084096, + "inactive_split_bytes_peak": 11043164160, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:30.694466Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472131072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472131072, + "device": 3, + "inactive_split_bytes_current": 5779524608, + "inactive_split_bytes_peak": 11043164160, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:29.638627Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5662084096, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 11043164160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110037565440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110037565440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T134330694880Z.pickle", + "timestamp_utc": "2026-08-01T13:43:30.802222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T134330694880Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T134330694880Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27361d35e54cb057f6ea3d133b07cba7c70c2355 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T134330694880Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:670df801f9631d2651c8b483707284b92659755df7147823aa054263d5bdb367 +size 6626255 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T134331996920Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T134331996920Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5143570db231d73544b5a1b95703de71ec92ba34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T134331996920Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159871488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472131072, + "device": 3, + "inactive_split_bytes_current": 5664181248, + "inactive_split_bytes_peak": 11043164160, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:31.996488Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472131072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472131072, + "device": 3, + "inactive_split_bytes_current": 5662084096, + "inactive_split_bytes_peak": 11043164160, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:30.818007Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5664181248, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11043164160, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110037565440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110037565440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T134331996920Z.pickle", + "timestamp_utc": "2026-08-01T13:43:32.103074Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T134331996920Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T134331996920Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe4c3a1a3d2c0903d087a632c0fb7a47c947e56a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T134331996920Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b3311f2d380bba787ae7147130cd3b6c0521181893b302691a3694937cf4dd5 +size 6622241 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T134333161188Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T134333161188Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8503db704c9f7af953e5fc1bf4d201433c8abbd5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T134333161188Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105159871488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472225280, + "device": 3, + "inactive_split_bytes_current": 5808884736, + "inactive_split_bytes_peak": 11436290048, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:33.160748Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104472225280, + "allocated_bytes_peak_delta": 94208, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472131072, + "device": 3, + "inactive_split_bytes_current": 5664181248, + "inactive_split_bytes_peak": 11043164160, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:32.118963Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5808884736, + "inactive_split_bytes_current_delta": 144703488, + "inactive_split_bytes_peak": 11436290048, + "inactive_split_bytes_peak_delta": 393125888, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110037565440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110037565440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T134333161188Z.pickle", + "timestamp_utc": "2026-08-01T13:43:33.267607Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T134333161188Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T134333161188Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..085addf3122d1a64815a85c5e13bacc55dc6d418 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T134333161188Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d538c2fa970496835842e2a5c4ded20abc13c6ccf0adbe88452090efba0b160 +size 6625726 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T134426036537Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T134426036537Z.json new file mode 100644 index 0000000000000000000000000000000000000000..770f512e9bc33ad8dc19a1477bc190476c206bef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T134426036537Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105159871488, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 104475502080, + "device": 3, + "inactive_split_bytes_current": 5771136000, + "inactive_split_bytes_peak": 11436290048, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:44:26.036065Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104475502080, + "allocated_bytes_peak_delta": 3276800, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105159871488, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104472225280, + "device": 3, + "inactive_split_bytes_current": 5808884736, + "inactive_split_bytes_peak": 11436290048, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110037565440, + "reserved_bytes_peak": 110037565440, + "segment_count_current": 518, + "segment_count_peak": 518, + "timestamp_utc": "2026-08-01T13:43:33.283374Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5771136000, + "inactive_split_bytes_current_delta": -37748736, + "inactive_split_bytes_peak": 11436290048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110037565440, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110037565440, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 518, + "segment_count_current_delta": 0, + "segment_count_peak": 518, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T134426036537Z.pickle", + "timestamp_utc": "2026-08-01T13:44:26.251169Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T134426036537Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T134426036537Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fef65cf132db85bebe258e076c21605432ac617f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T134426036537Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db0d6cb06b0c6697720b10ca5313a38d7b67c80ec250de2b1ebd51904f4b0148 +size 9722557 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T134323149675Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T134323149675Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bad46aac8a6db32b6ccd2a61b7bcdc99c6665f75 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T134323149675Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441442304, + "active_bytes_current_delta": 37209544704, + "active_bytes_peak": 81526473728, + "active_bytes_peak_delta": 46059695104, + "after": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 81526473728, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 80833871872, + "device": 4, + "inactive_split_bytes_current": 5115432960, + "inactive_split_bytes_peak": 8753003520, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 97987330048, + "reserved_bytes_peak": 97987330048, + "segment_count_current": 469, + "segment_count_peak": 469, + "timestamp_utc": "2026-08-01T13:43:23.149127Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441442304, + "allocated_bytes_current_delta": 37209544704, + "allocated_bytes_peak": 80833871872, + "allocated_bytes_peak_delta": 45367093248, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:42:15.516436Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5115432960, + "inactive_split_bytes_current_delta": 4983056384, + "inactive_split_bytes_peak": 8753003520, + "inactive_split_bytes_peak_delta": 8599593984, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 97987330048, + "reserved_bytes_current_delta": 58743324672, + "reserved_bytes_peak": 97987330048, + "reserved_bytes_peak_delta": 58743324672, + "segment_count_current": 469, + "segment_count_current_delta": 398, + "segment_count_peak": 469, + "segment_count_peak_delta": 398, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T134323149675Z.pickle", + "timestamp_utc": "2026-08-01T13:43:23.265961Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T134323149675Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T134323149675Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..46c66dc2f1db48c08488c64e5f29e8441112b8ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T134323149675Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a51309d6a3598961533ac89c1729bc2e1419a25cd1f90550cf5e2c8bd8c3983 +size 5952337 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T134324308968Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T134324308968Z.json new file mode 100644 index 0000000000000000000000000000000000000000..66441f28be15690962aaccb11aa74cc8ca216144 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T134324308968Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441442304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105163213824, + "active_bytes_peak_delta": 23636740096, + "after": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471660544, + "device": 4, + "inactive_split_bytes_current": 6237409280, + "inactive_split_bytes_peak": 11820209152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109093847040, + "reserved_bytes_peak": 109093847040, + "segment_count_current": 521, + "segment_count_peak": 521, + "timestamp_utc": "2026-08-01T13:43:24.308557Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441442304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471660544, + "allocated_bytes_peak_delta": 23637788672, + "before": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 81526473728, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 80833871872, + "device": 4, + "inactive_split_bytes_current": 5115432960, + "inactive_split_bytes_peak": 8753003520, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 97987330048, + "reserved_bytes_peak": 97987330048, + "segment_count_current": 469, + "segment_count_peak": 469, + "timestamp_utc": "2026-08-01T13:43:23.282570Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6237409280, + "inactive_split_bytes_current_delta": 1121976320, + "inactive_split_bytes_peak": 11820209152, + "inactive_split_bytes_peak_delta": 3067205632, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109093847040, + "reserved_bytes_current_delta": 11106516992, + "reserved_bytes_peak": 109093847040, + "reserved_bytes_peak_delta": 11106516992, + "segment_count_current": 521, + "segment_count_current_delta": 52, + "segment_count_peak": 521, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T134324308968Z.pickle", + "timestamp_utc": "2026-08-01T13:43:24.426895Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T134324308968Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T134324308968Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..90da64579510a619d7a1bd56028d3d7c3c5e0ffb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T134324308968Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e06302f81ba823454912b2bf121176d06231c50785395db9c12120292e3d61a7 +size 6631973 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T134325474433Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T134325474433Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b5ad55e903a99ee7dfcab789853d36d99987539c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T134325474433Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441442304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105163213824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471987200, + "device": 4, + "inactive_split_bytes_current": 5203513344, + "inactive_split_bytes_peak": 11820209152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109093847040, + "reserved_bytes_peak": 109093847040, + "segment_count_current": 521, + "segment_count_peak": 521, + "timestamp_utc": "2026-08-01T13:43:25.474033Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441442304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471987200, + "allocated_bytes_peak_delta": 326656, + "before": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471660544, + "device": 4, + "inactive_split_bytes_current": 6237409280, + "inactive_split_bytes_peak": 11820209152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 109093847040, + "reserved_bytes_peak": 109093847040, + "segment_count_current": 521, + "segment_count_peak": 521, + "timestamp_utc": "2026-08-01T13:43:24.442961Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5203513344, + "inactive_split_bytes_current_delta": -1033895936, + "inactive_split_bytes_peak": 11820209152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109093847040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109093847040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 521, + "segment_count_current_delta": 0, + "segment_count_peak": 521, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T134325474433Z.pickle", + "timestamp_utc": "2026-08-01T13:43:25.583978Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T134325474433Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T134325474433Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c986d1f27aac6d955d37b3f9081fba35592ca2b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T134325474433Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5469948cd44bc06597450baa6dd4e5b6469bb0c0c91cd3b3547e9bb99bdd2d5c +size 6629497 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T134326899499Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T134326899499Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e8dcca8edd8ed1f9f932f65c2ba3e8b1eecaf14e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T134326899499Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441442304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105163213824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471987200, + "device": 4, + "inactive_split_bytes_current": 5648109568, + "inactive_split_bytes_peak": 11820209152, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:43:26.899101Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441442304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471987200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471987200, + "device": 4, + "inactive_split_bytes_current": 5203513344, + "inactive_split_bytes_peak": 11820209152, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 109093847040, + "reserved_bytes_peak": 109093847040, + "segment_count_current": 521, + "segment_count_peak": 521, + "timestamp_utc": "2026-08-01T13:43:25.602173Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5648109568, + "inactive_split_bytes_current_delta": 444596224, + "inactive_split_bytes_peak": 11820209152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109506985984, + "reserved_bytes_current_delta": 413138944, + "reserved_bytes_peak": 109506985984, + "reserved_bytes_peak_delta": 413138944, + "segment_count_current": 522, + "segment_count_current_delta": 1, + "segment_count_peak": 522, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T134326899499Z.pickle", + "timestamp_utc": "2026-08-01T13:43:27.013902Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T134326899499Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T134326899499Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0f2c1f036752194e7990f6fe291acee8dce7352b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T134326899499Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73a54e9c194c0b99d42f99f1e170df1415c14d4327c80230f72f714d98c40439 +size 6628998 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T134328352806Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T134328352806Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42e8a937f28fe96aa2d10152d625f440e918dcfa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T134328352806Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441442304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105163213824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471987200, + "device": 4, + "inactive_split_bytes_current": 5855727616, + "inactive_split_bytes_peak": 11820209152, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:43:28.352382Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441442304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471987200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471987200, + "device": 4, + "inactive_split_bytes_current": 5648109568, + "inactive_split_bytes_peak": 11820209152, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:43:27.029067Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5855727616, + "inactive_split_bytes_current_delta": 207618048, + "inactive_split_bytes_peak": 11820209152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109506985984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109506985984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T134328352806Z.pickle", + "timestamp_utc": "2026-08-01T13:43:28.463164Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T134328352806Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T134328352806Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..95b4324cf1855fc404b9a95ef447ef6aae1de239 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T134328352806Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b0726b6734f6189ffa66c089b740a451ba6934b0c85a9c98d1523b8a1f3aac5 +size 6628193 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T134329515148Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T134329515148Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5769646da3d30bf664341e8a221d2d324ee1efc7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T134329515148Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441442304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105163213824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471987200, + "device": 4, + "inactive_split_bytes_current": 6029791232, + "inactive_split_bytes_peak": 11820209152, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:43:29.514754Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441442304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471987200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471987200, + "device": 4, + "inactive_split_bytes_current": 5855727616, + "inactive_split_bytes_peak": 11820209152, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:43:28.478319Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6029791232, + "inactive_split_bytes_current_delta": 174063616, + "inactive_split_bytes_peak": 11820209152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109506985984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109506985984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T134329515148Z.pickle", + "timestamp_utc": "2026-08-01T13:43:29.621800Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T134329515148Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T134329515148Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f0c0076f90615b107c4bbc7a2c026e9d1df49a64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T134329515148Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe4bb421506f66a3a805997b996019eb880d994bde51777f66433076551e95fc +size 6628006 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T134330694820Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T134330694820Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5530bb2175160aa95a32187183311133fdfa6895 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T134330694820Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441442304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105163213824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471987200, + "device": 4, + "inactive_split_bytes_current": 5438394368, + "inactive_split_bytes_peak": 11820209152, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:43:30.694414Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441442304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471987200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471987200, + "device": 4, + "inactive_split_bytes_current": 6029791232, + "inactive_split_bytes_peak": 11820209152, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:43:29.636749Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5438394368, + "inactive_split_bytes_current_delta": -591396864, + "inactive_split_bytes_peak": 11820209152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109506985984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109506985984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T134330694820Z.pickle", + "timestamp_utc": "2026-08-01T13:43:30.802650Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T134330694820Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T134330694820Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ffe63cb67987b25f1df127f0c14e1027d049044b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T134330694820Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:645138dae7599a44d44fb4548e616c5729b66ae9e5b6ffa716c4e63b33c41e5d +size 6627669 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T134331996740Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T134331996740Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a4eb4af750f0898ab40570295245b68fb865301b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T134331996740Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441442304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105163213824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471987200, + "device": 4, + "inactive_split_bytes_current": 5704732672, + "inactive_split_bytes_peak": 11820209152, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:43:31.996338Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441442304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471987200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471987200, + "device": 4, + "inactive_split_bytes_current": 5438394368, + "inactive_split_bytes_peak": 11820209152, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:43:30.817800Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5704732672, + "inactive_split_bytes_current_delta": 266338304, + "inactive_split_bytes_peak": 11820209152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109506985984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109506985984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T134331996740Z.pickle", + "timestamp_utc": "2026-08-01T13:43:32.103735Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T134331996740Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T134331996740Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c8eae3283a55820edbdad1394ddb2289e10f0100 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T134331996740Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e03cd2531913be292a287c988919bbf284eee33eea82d7e45c85621ffad15ee1 +size 6627551 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T134333161153Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T134333161153Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a297d6b02872cddf2b4e0a3d457808c1f1257f58 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T134333161153Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441442304, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105163213824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104476050432, + "device": 4, + "inactive_split_bytes_current": 5174153216, + "inactive_split_bytes_peak": 11820209152, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:43:33.160731Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441442304, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104476050432, + "allocated_bytes_peak_delta": 4063232, + "before": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104471987200, + "device": 4, + "inactive_split_bytes_current": 5704732672, + "inactive_split_bytes_peak": 11820209152, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:43:32.118800Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5174153216, + "inactive_split_bytes_current_delta": -530579456, + "inactive_split_bytes_peak": 11820209152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109506985984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109506985984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T134333161153Z.pickle", + "timestamp_utc": "2026-08-01T13:43:33.269779Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T134333161153Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T134333161153Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bcb777a9fe69e0043394e39d94f2a1e76d068acb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T134333161153Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:852946adb16fbd8922f2bc17af7e70bd5d1fcec02857e2a797d3a9e8d272c7af +size 6627502 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T134426036532Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T134426036532Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ffdbff0f09accb679a1fc03519c980cd2f14e7a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T134426036532Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61167153152, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105163213824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61167153152, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 61167153152, + "allocated_bytes_peak": 104476050432, + "device": 4, + "inactive_split_bytes_current": 4641476608, + "inactive_split_bytes_peak": 11820209152, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:44:26.036054Z", + "world_size": 8 + }, + "allocated_bytes_current": 61167153152, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104476050432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441442304, + "active_bytes_peak": 105163213824, + "allocated_bytes_current": 72441442304, + "allocated_bytes_peak": 104476050432, + "device": 4, + "inactive_split_bytes_current": 5174153216, + "inactive_split_bytes_peak": 11820209152, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 109506985984, + "reserved_bytes_peak": 109506985984, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T13:43:33.284833Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4641476608, + "inactive_split_bytes_current_delta": -532676608, + "inactive_split_bytes_peak": 11820209152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 109506985984, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109506985984, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T134426036532Z.pickle", + "timestamp_utc": "2026-08-01T13:44:26.261928Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T134426036532Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T134426036532Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de7a3e3553ad76a66fa3cb575778eed13cdb4d7f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T134426036532Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d681396a3bcb34e156518214d7e29371458b89776924de0643ae32a7bf59aa1d +size 10526830 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T134323149825Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T134323149825Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c359716f60a365491beb9c6de9ddc27bc32db02 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T134323149825Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 81171858432, + "active_bytes_peak_delta": 45705522176, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 81171858432, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80831578112, + "device": 5, + "inactive_split_bytes_current": 4850486272, + "inactive_split_bytes_peak": 9441873408, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 98150907904, + "reserved_bytes_peak": 98150907904, + "segment_count_current": 464, + "segment_count_peak": 464, + "timestamp_utc": "2026-08-01T13:43:23.149233Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 80831578112, + "allocated_bytes_peak_delta": 45365241856, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:42:15.516935Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4850486272, + "inactive_split_bytes_current_delta": 4717667328, + "inactive_split_bytes_peak": 9441873408, + "inactive_split_bytes_peak_delta": 9281487360, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 98150907904, + "reserved_bytes_current_delta": 58906902528, + "reserved_bytes_peak": 98150907904, + "reserved_bytes_peak_delta": 58906902528, + "segment_count_current": 464, + "segment_count_current_delta": 393, + "segment_count_peak": 464, + "segment_count_peak_delta": 393, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T134323149825Z.pickle", + "timestamp_utc": "2026-08-01T13:43:23.269143Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T134323149825Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T134323149825Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..764b4b10ae1d0131f7d9d45f2e931f50152f9413 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T134323149825Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1005494c288f7064120da112edf0cf60cec00dfde23fdbd39b1dcd8f046c9fac +size 5942928 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T134324309070Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T134324309070Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a4d41185f0c335657e0c9120c52ed2a65416f505 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T134324309070Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104808376832, + "active_bytes_peak_delta": 23636518400, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104808376832, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104468096512, + "device": 5, + "inactive_split_bytes_current": 5966171136, + "inactive_split_bytes_peak": 11354891264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109741867008, + "reserved_bytes_peak": 109741867008, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:24.308655Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104468096512, + "allocated_bytes_peak_delta": 23636518400, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 81171858432, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80831578112, + "device": 5, + "inactive_split_bytes_current": 4850486272, + "inactive_split_bytes_peak": 9441873408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 98150907904, + "reserved_bytes_peak": 98150907904, + "segment_count_current": 464, + "segment_count_peak": 464, + "timestamp_utc": "2026-08-01T13:43:23.288347Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5966171136, + "inactive_split_bytes_current_delta": 1115684864, + "inactive_split_bytes_peak": 11354891264, + "inactive_split_bytes_peak_delta": 1913017856, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109741867008, + "reserved_bytes_current_delta": 11590959104, + "reserved_bytes_peak": 109741867008, + "reserved_bytes_peak_delta": 11590959104, + "segment_count_current": 516, + "segment_count_current_delta": 52, + "segment_count_peak": 516, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T134324309070Z.pickle", + "timestamp_utc": "2026-08-01T13:43:24.427375Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T134324309070Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T134324309070Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3602540918d114ec148127db0056ac10183fce72 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T134324309070Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdbe18c69f905384eda3307009493b746005822a5e38c359fcbdea39eeff2cbb +size 6630788 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T134325474553Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T134325474553Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8cb4f81739a0fd26127386a179beed4caaaac3c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T134325474553Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104811326464, + "active_bytes_peak_delta": 2949632, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104811326464, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471046144, + "device": 5, + "inactive_split_bytes_current": 5202807808, + "inactive_split_bytes_peak": 11354891264, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109741867008, + "reserved_bytes_peak": 109741867008, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:25.474154Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471046144, + "allocated_bytes_peak_delta": 2949632, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104808376832, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104468096512, + "device": 5, + "inactive_split_bytes_current": 5966171136, + "inactive_split_bytes_peak": 11354891264, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 109741867008, + "reserved_bytes_peak": 109741867008, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:24.444058Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5202807808, + "inactive_split_bytes_current_delta": -763363328, + "inactive_split_bytes_peak": 11354891264, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109741867008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109741867008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 516, + "segment_count_current_delta": 0, + "segment_count_peak": 516, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T134325474553Z.pickle", + "timestamp_utc": "2026-08-01T13:43:25.584732Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T134325474553Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T134325474553Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0414c26124d9388fd0b43820dc0688c41cbf2724 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T134325474553Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ee12b1c42d4429103267fd7fffee69afc0fe698a3201ff8fcc6358a0e43f9fd +size 6626711 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T134326899631Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T134326899631Z.json new file mode 100644 index 0000000000000000000000000000000000000000..86483407463da9fb7b6d54fd1d671e37a4e109cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T134326899631Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104811326464, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104811326464, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471046144, + "device": 5, + "inactive_split_bytes_current": 5439785984, + "inactive_split_bytes_peak": 11808514048, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:26.899218Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471046144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104811326464, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471046144, + "device": 5, + "inactive_split_bytes_current": 5202807808, + "inactive_split_bytes_peak": 11354891264, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 109741867008, + "reserved_bytes_peak": 109741867008, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:25.600886Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5439785984, + "inactive_split_bytes_current_delta": 236978176, + "inactive_split_bytes_peak": 11808514048, + "inactive_split_bytes_peak_delta": 453622784, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109743964160, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 109743964160, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 517, + "segment_count_current_delta": 1, + "segment_count_peak": 517, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T134326899631Z.pickle", + "timestamp_utc": "2026-08-01T13:43:27.010664Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T134326899631Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T134326899631Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1cd05032747ab40b9c5967122edf0f9ccc8f7b8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T134326899631Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82186d26e69dc1e54f287c2c04e70c704af6beae2e649c999541d2d8b4e1f7ff +size 6626752 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T134328352896Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T134328352896Z.json new file mode 100644 index 0000000000000000000000000000000000000000..64e93ce0bd871a8a803f400e67573588307fcb12 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T134328352896Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104814038016, + "active_bytes_peak_delta": 2711552, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104814038016, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104473757696, + "device": 5, + "inactive_split_bytes_current": 4760308736, + "inactive_split_bytes_peak": 11808514048, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:28.352489Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104473757696, + "allocated_bytes_peak_delta": 2711552, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104811326464, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471046144, + "device": 5, + "inactive_split_bytes_current": 5439785984, + "inactive_split_bytes_peak": 11808514048, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:27.026635Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4760308736, + "inactive_split_bytes_current_delta": -679477248, + "inactive_split_bytes_peak": 11808514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109743964160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109743964160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 517, + "segment_count_current_delta": 0, + "segment_count_peak": 517, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T134328352896Z.pickle", + "timestamp_utc": "2026-08-01T13:43:28.466683Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T134328352896Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T134328352896Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..708df2d918d896130bdbb0eb336fecb74a64706c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T134328352896Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23d9149f2132a95c87f262111d5a3e67b0b3ba72f44d8014cf942be9e1f79c9d +size 6626164 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T134329515245Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T134329515245Z.json new file mode 100644 index 0000000000000000000000000000000000000000..582d5f23db99ef3c4c188a4d821d02a9bc08cc8d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T134329515245Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104814038016, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104814038016, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104473757696, + "device": 5, + "inactive_split_bytes_current": 5469146112, + "inactive_split_bytes_peak": 11808514048, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:29.514839Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104473757696, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104814038016, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104473757696, + "device": 5, + "inactive_split_bytes_current": 4760308736, + "inactive_split_bytes_peak": 11808514048, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:28.484890Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5469146112, + "inactive_split_bytes_current_delta": 708837376, + "inactive_split_bytes_peak": 11808514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109743964160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109743964160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 517, + "segment_count_current_delta": 0, + "segment_count_peak": 517, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T134329515245Z.pickle", + "timestamp_utc": "2026-08-01T13:43:29.627470Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T134329515245Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T134329515245Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..46e2105f248be00ea1617248063ad3c0c51e9232 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T134329515245Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81a9a790b7eb1d62b3f3baaae3e81178be61fe2c2e1eb5d85f1ba059cea96a87 +size 6625684 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T134330694946Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T134330694946Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d8735d1b9b3facb42288288e65c134ef6f1c11be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T134330694946Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104814275072, + "active_bytes_peak_delta": 237056, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104814275072, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104473994752, + "device": 5, + "inactive_split_bytes_current": 5230070784, + "inactive_split_bytes_peak": 11808514048, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:30.694532Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104473994752, + "allocated_bytes_peak_delta": 237056, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104814038016, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104473757696, + "device": 5, + "inactive_split_bytes_current": 5469146112, + "inactive_split_bytes_peak": 11808514048, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:29.643244Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5230070784, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 11808514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109743964160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109743964160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 517, + "segment_count_current_delta": 0, + "segment_count_peak": 517, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T134330694946Z.pickle", + "timestamp_utc": "2026-08-01T13:43:30.802689Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T134330694946Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T134330694946Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c9321f3d18140b4fbeab999c25637eee61b315e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T134330694946Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e31bc149b563b26604d01714fb9ae96d5194eee783a96b9bf232301b64edeca4 +size 6625743 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T134331996857Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T134331996857Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9e15b3fe39176e86e27930d269f7eacbcac0f9ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T134331996857Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104814275072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104814275072, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104473994752, + "device": 5, + "inactive_split_bytes_current": 5469146112, + "inactive_split_bytes_peak": 11808514048, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:31.996439Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104473994752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104814275072, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104473994752, + "device": 5, + "inactive_split_bytes_current": 5230070784, + "inactive_split_bytes_peak": 11808514048, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:30.818170Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5469146112, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 11808514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109743964160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109743964160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 517, + "segment_count_current_delta": 0, + "segment_count_peak": 517, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T134331996857Z.pickle", + "timestamp_utc": "2026-08-01T13:43:32.108047Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T134331996857Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T134331996857Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..454caa0df037eb2b7a283aed344caca461e17d6a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T134331996857Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d686af99f2534e6a1db183d0b3b10a4b1579e738d80360f361fe74ee7553b6c2 +size 6625680 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T134333161246Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T134333161246Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2e687d2912ad556db56f200c07a4b01305e3921a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T134333161246Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 104814275072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104814275072, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104473994752, + "device": 5, + "inactive_split_bytes_current": 5320248320, + "inactive_split_bytes_peak": 11808514048, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:33.160821Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104473994752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104814275072, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104473994752, + "device": 5, + "inactive_split_bytes_current": 5469146112, + "inactive_split_bytes_peak": 11808514048, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:32.124237Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5320248320, + "inactive_split_bytes_current_delta": -148897792, + "inactive_split_bytes_peak": 11808514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109743964160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109743964160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 517, + "segment_count_current_delta": 0, + "segment_count_peak": 517, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T134333161246Z.pickle", + "timestamp_utc": "2026-08-01T13:43:33.269003Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T134333161246Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T134333161246Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a84e97807f7db84b071b98bb2d756fc8e8cc8231 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T134333161246Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83716a22aa23c9e1d60129ed9132b06a497351cd3f97120ebf1e3c31a64f36b7 +size 6625566 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T134426036626Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T134426036626Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d76ef13e797f8299f39520a153d71e2c0ca85e61 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T134426036626Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 104814275072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 104814275072, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 104473994752, + "device": 5, + "inactive_split_bytes_current": 6136040448, + "inactive_split_bytes_peak": 11808514048, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:44:26.036049Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104473994752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 104814275072, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104473994752, + "device": 5, + "inactive_split_bytes_current": 5320248320, + "inactive_split_bytes_peak": 11808514048, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 109743964160, + "reserved_bytes_peak": 109743964160, + "segment_count_current": 517, + "segment_count_peak": 517, + "timestamp_utc": "2026-08-01T13:43:33.284450Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6136040448, + "inactive_split_bytes_current_delta": 815792128, + "inactive_split_bytes_peak": 11808514048, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 109743964160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109743964160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 517, + "segment_count_current_delta": 0, + "segment_count_peak": 517, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T134426036626Z.pickle", + "timestamp_utc": "2026-08-01T13:44:26.247906Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T134426036626Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T134426036626Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1801ce08985eecbe9b096472e209e222e58320b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T134426036626Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfe7a2fdca2ff5049a97487a2348a7e3540c8ab24d201541a5f9d31cdd27ac0f +size 9804038 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T134323149751Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T134323149751Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1bb4d1d85f60932f876b3d6f3fbe74de971b7b59 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T134323149751Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 81527659520, + "active_bytes_peak_delta": 46061323264, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 81527659520, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80835701248, + "device": 6, + "inactive_split_bytes_current": 5303471104, + "inactive_split_bytes_peak": 9070967808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98662612992, + "reserved_bytes_peak": 98662612992, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T13:43:23.149140Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 80835701248, + "allocated_bytes_peak_delta": 45369364992, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:42:15.516875Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5303471104, + "inactive_split_bytes_current_delta": 5170652160, + "inactive_split_bytes_peak": 9070967808, + "inactive_split_bytes_peak_delta": 8910581760, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98662612992, + "reserved_bytes_current_delta": 59418607616, + "reserved_bytes_peak": 98662612992, + "reserved_bytes_peak_delta": 59418607616, + "segment_count_current": 466, + "segment_count_current_delta": 395, + "segment_count_peak": 466, + "segment_count_peak_delta": 395, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T134323149751Z.pickle", + "timestamp_utc": "2026-08-01T13:43:23.271084Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T134323149751Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T134323149751Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..174a15715ba1a17f12ae627e085cb10465314d63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T134323149751Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d12225220718002c00cf1b2115f477e34b3e133802bf80d1dcafecd2e3b62c28 +size 5943375 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T134324309011Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T134324309011Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5c965b1f651a9eab87c98b5b62a0c353b8af2312 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T134324309011Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105152269312, + "active_bytes_peak_delta": 23624609792, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105152269312, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471135744, + "device": 6, + "inactive_split_bytes_current": 5276208128, + "inactive_split_bytes_peak": 11518157824, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108785565696, + "reserved_bytes_peak": 108785565696, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:43:24.308583Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471135744, + "allocated_bytes_peak_delta": 23635434496, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 81527659520, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 80835701248, + "device": 6, + "inactive_split_bytes_current": 5303471104, + "inactive_split_bytes_peak": 9070967808, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98662612992, + "reserved_bytes_peak": 98662612992, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T13:43:23.291464Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5276208128, + "inactive_split_bytes_current_delta": -27262976, + "inactive_split_bytes_peak": 11518157824, + "inactive_split_bytes_peak_delta": 2447190016, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108785565696, + "reserved_bytes_current_delta": 10122952704, + "reserved_bytes_peak": 108785565696, + "reserved_bytes_peak_delta": 10122952704, + "segment_count_current": 512, + "segment_count_current_delta": 46, + "segment_count_peak": 512, + "segment_count_peak_delta": 46, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T134324309011Z.pickle", + "timestamp_utc": "2026-08-01T13:43:24.425203Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T134324309011Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T134324309011Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b727c2dd4a76d99facb764eaadbdfb933540fb5b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T134324309011Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4af0518b3ff45b387321dbe4304b51c67f6f63ae40d92c748868bf228993ccf +size 5962334 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T134325474472Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T134325474472Z.json new file mode 100644 index 0000000000000000000000000000000000000000..189c953e04f2024310c3b9c4403c1deeb980f129 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T134325474472Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105152269312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105152269312, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471135744, + "device": 6, + "inactive_split_bytes_current": 5425105920, + "inactive_split_bytes_peak": 11518157824, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108785565696, + "reserved_bytes_peak": 108785565696, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:43:25.474068Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471135744, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105152269312, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471135744, + "device": 6, + "inactive_split_bytes_current": 5276208128, + "inactive_split_bytes_peak": 11518157824, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108785565696, + "reserved_bytes_peak": 108785565696, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:43:24.439374Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5425105920, + "inactive_split_bytes_current_delta": 148897792, + "inactive_split_bytes_peak": 11518157824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108785565696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108785565696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 512, + "segment_count_current_delta": 0, + "segment_count_peak": 512, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T134325474472Z.pickle", + "timestamp_utc": "2026-08-01T13:43:25.580032Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T134325474472Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T134325474472Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a699b67cc7f3aaa9cfeb2940aa5260f5d086a5ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T134325474472Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e751ebdd23401e51a9fcb4cd265643395839a4c3fe34496f08f0a3a1c93816b +size 6070723 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T134326899557Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T134326899557Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db873804d132f5123b1ff24bb7fd19bb12939876 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T134326899557Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105152269312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105152269312, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471135744, + "device": 6, + "inactive_split_bytes_current": 5427203072, + "inactive_split_bytes_peak": 11518157824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109962067968, + "reserved_bytes_peak": 109962067968, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T13:43:26.899138Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471135744, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105152269312, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471135744, + "device": 6, + "inactive_split_bytes_current": 5425105920, + "inactive_split_bytes_peak": 11518157824, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108785565696, + "reserved_bytes_peak": 108785565696, + "segment_count_current": 512, + "segment_count_peak": 512, + "timestamp_utc": "2026-08-01T13:43:25.593772Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5427203072, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11518157824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109962067968, + "reserved_bytes_current_delta": 1176502272, + "reserved_bytes_peak": 109962067968, + "reserved_bytes_peak_delta": 1176502272, + "segment_count_current": 515, + "segment_count_current_delta": 3, + "segment_count_peak": 515, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T134326899557Z.pickle", + "timestamp_utc": "2026-08-01T13:43:27.012720Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T134326899557Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T134326899557Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a1e4747b3ee93063c891279b2bb8671ecd0d44b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T134326899557Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78788dbf629ae7ec3ebf2cd91ddcd9e9f921c8ac22683b0645f3d332641d3a64 +size 6616971 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T134328352817Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T134328352817Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aa63f12dba8e1edd98035f6cc8dacc439b85e3f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T134328352817Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105152269312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105152269312, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471135744, + "device": 6, + "inactive_split_bytes_current": 5636918272, + "inactive_split_bytes_peak": 11555332096, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109964165120, + "reserved_bytes_peak": 109964165120, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:28.352407Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471135744, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105152269312, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471135744, + "device": 6, + "inactive_split_bytes_current": 5427203072, + "inactive_split_bytes_peak": 11518157824, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 109962067968, + "reserved_bytes_peak": 109962067968, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T13:43:27.026668Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5636918272, + "inactive_split_bytes_current_delta": 209715200, + "inactive_split_bytes_peak": 11555332096, + "inactive_split_bytes_peak_delta": 37174272, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109964165120, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 109964165120, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 516, + "segment_count_current_delta": 1, + "segment_count_peak": 516, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T134328352817Z.pickle", + "timestamp_utc": "2026-08-01T13:43:28.464550Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T134328352817Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T134328352817Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4643508a31e11ec9c349dabd51c17e3dabbca669 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T134328352817Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef0127a9923382015b3179beb182a0d7dde8fc7281cca9f0fb0c2647e6b03692 +size 6626729 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T134329515159Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T134329515159Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a3ba1a2ac78b0dfda04e1e7fd1926d94ccb9d06 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T134329515159Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105152269312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105152269312, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471135744, + "device": 6, + "inactive_split_bytes_current": 5337025536, + "inactive_split_bytes_peak": 11555332096, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109964165120, + "reserved_bytes_peak": 109964165120, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:29.514752Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471135744, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105152269312, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471135744, + "device": 6, + "inactive_split_bytes_current": 5636918272, + "inactive_split_bytes_peak": 11555332096, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 109964165120, + "reserved_bytes_peak": 109964165120, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:28.480444Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5337025536, + "inactive_split_bytes_current_delta": -299892736, + "inactive_split_bytes_peak": 11555332096, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109964165120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109964165120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 516, + "segment_count_current_delta": 0, + "segment_count_peak": 516, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T134329515159Z.pickle", + "timestamp_utc": "2026-08-01T13:43:29.621143Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T134329515159Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T134329515159Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ceacc901e7cd66cf8b6dd02c7496da4f8de0775 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T134329515159Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a019332e5f4c9c44c08f645011b8ba020cfdcfb07ef725541f8942a734d1082 +size 6623311 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T134330694956Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T134330694956Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c1e3fdc3f93fae4c597358389e9e265566f09b58 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T134330694956Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105152269312, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105152269312, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471812096, + "device": 6, + "inactive_split_bytes_current": 5485923328, + "inactive_split_bytes_peak": 11555332096, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109964165120, + "reserved_bytes_peak": 109964165120, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:30.694509Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471812096, + "allocated_bytes_peak_delta": 676352, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105152269312, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471135744, + "device": 6, + "inactive_split_bytes_current": 5337025536, + "inactive_split_bytes_peak": 11555332096, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 109964165120, + "reserved_bytes_peak": 109964165120, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:29.634437Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5485923328, + "inactive_split_bytes_current_delta": 148897792, + "inactive_split_bytes_peak": 11555332096, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109964165120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109964165120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 516, + "segment_count_current_delta": 0, + "segment_count_peak": 516, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T134330694956Z.pickle", + "timestamp_utc": "2026-08-01T13:43:30.800860Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T134330694956Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T134330694956Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..837f4335b8ac73bd34b15a9bc95898ce94b7f7db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T134330694956Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d2dc20a1d305878da0a5322237f789668a878021e341e29c9e5dc2b0654bce0 +size 6623014 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T134331996784Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T134331996784Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b956bee5b22b5f27b4ced1395cfd8bf8db733624 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T134331996784Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105156766208, + "active_bytes_peak_delta": 4496896, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105156766208, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471812096, + "device": 6, + "inactive_split_bytes_current": 5309762560, + "inactive_split_bytes_peak": 11555332096, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109964165120, + "reserved_bytes_peak": 109964165120, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:31.996379Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471812096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105152269312, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471812096, + "device": 6, + "inactive_split_bytes_current": 5485923328, + "inactive_split_bytes_peak": 11555332096, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 109964165120, + "reserved_bytes_peak": 109964165120, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:30.813823Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5309762560, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 11555332096, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109964165120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109964165120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 516, + "segment_count_current_delta": 0, + "segment_count_peak": 516, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T134331996784Z.pickle", + "timestamp_utc": "2026-08-01T13:43:32.099521Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T134331996784Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T134331996784Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..506c3c2822a0b1364c3ee290f376c9ab8f69ebe0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T134331996784Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:196a34e62b0caa5aa7b303319209ac096f0bdaa1716c106275b9a75530d27a8c +size 6622337 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T134333161149Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T134333161149Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2179add5f51cc892973a5db61ead62f62ff600d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T134333161149Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105156766208, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105156766208, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471812096, + "device": 6, + "inactive_split_bytes_current": 5485923328, + "inactive_split_bytes_peak": 11555332096, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109964165120, + "reserved_bytes_peak": 109964165120, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:33.160731Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104471812096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105156766208, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471812096, + "device": 6, + "inactive_split_bytes_current": 5309762560, + "inactive_split_bytes_peak": 11555332096, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 109964165120, + "reserved_bytes_peak": 109964165120, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:32.112399Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5485923328, + "inactive_split_bytes_current_delta": 176160768, + "inactive_split_bytes_peak": 11555332096, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109964165120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109964165120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 516, + "segment_count_current_delta": 0, + "segment_count_peak": 516, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T134333161149Z.pickle", + "timestamp_utc": "2026-08-01T13:43:33.263051Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T134333161149Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T134333161149Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eea3a96c46085badfe286a6a235f5b6a177392c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T134333161149Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ae17cacac345fe4e768a9bf28046a3e5914bb4c488539e4fc55ac205863633c +size 6616159 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T134426036595Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T134426036595Z.json new file mode 100644 index 0000000000000000000000000000000000000000..72c638ddaf73364a94f1a2ee477d4bb34b51060e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T134426036595Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105156766208, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 105156766208, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 104471812096, + "device": 6, + "inactive_split_bytes_current": 6123457536, + "inactive_split_bytes_peak": 11555332096, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 109964165120, + "reserved_bytes_peak": 109964165120, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:44:26.036051Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104471812096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105156766208, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 104471812096, + "device": 6, + "inactive_split_bytes_current": 5485923328, + "inactive_split_bytes_peak": 11555332096, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 109964165120, + "reserved_bytes_peak": 109964165120, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:33.276063Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6123457536, + "inactive_split_bytes_current_delta": 637534208, + "inactive_split_bytes_peak": 11555332096, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 109964165120, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 109964165120, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 516, + "segment_count_current_delta": 0, + "segment_count_peak": 516, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T134426036595Z.pickle", + "timestamp_utc": "2026-08-01T13:44:26.236146Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T134426036595Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T134426036595Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c9d182b58858cceb2dc2f4cc2954e4910c97c032 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T134426036595Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5427d17c89b96177dc2818d95d2583ab7f9d25cb51101f578d8ea8ef9b0962b6 +size 9722949 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T134323149677Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T134323149677Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0b5cca6e39fad7570d2423ed30a087476da4a220 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T134323149677Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 37208595968, + "active_bytes_peak": 81876567040, + "active_bytes_peak_delta": 46410230784, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 81876567040, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 80833932288, + "device": 7, + "inactive_split_bytes_current": 5141989888, + "inactive_split_bytes_peak": 9272692224, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 97945387008, + "reserved_bytes_peak": 97945387008, + "segment_count_current": 464, + "segment_count_peak": 464, + "timestamp_utc": "2026-08-01T13:43:23.149123Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 37208595968, + "allocated_bytes_peak": 80833932288, + "allocated_bytes_peak_delta": 45367596032, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:42:15.516877Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5141989888, + "inactive_split_bytes_current_delta": 5009170944, + "inactive_split_bytes_peak": 9272692224, + "inactive_split_bytes_peak_delta": 9112306176, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 97945387008, + "reserved_bytes_current_delta": 58701381632, + "reserved_bytes_peak": 97945387008, + "reserved_bytes_peak_delta": 58701381632, + "segment_count_current": 464, + "segment_count_current_delta": 393, + "segment_count_peak": 464, + "segment_count_peak_delta": 393, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T134323149677Z.pickle", + "timestamp_utc": "2026-08-01T13:43:23.269569Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T134323149677Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T134323149677Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..808c5a69eaf8f67ee039c4c1bd3def3183c1fbdd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T134323149677Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e50b7af361df18a0fc7389ba9cd7b2e162844a84d17302a02a5af9caa302621 +size 5944588 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T134324308950Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T134324308950Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c57f7aca87d81a8f473d71a6f52870ccc7ad2b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T134324308950Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105154407936, + "active_bytes_peak_delta": 23277840896, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105154407936, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104465192960, + "device": 7, + "inactive_split_bytes_current": 5997627904, + "inactive_split_bytes_peak": 11371053568, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 109108527104, + "reserved_bytes_peak": 109108527104, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:24.308549Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104465192960, + "allocated_bytes_peak_delta": 23631260672, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 81876567040, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 80833932288, + "device": 7, + "inactive_split_bytes_current": 5141989888, + "inactive_split_bytes_peak": 9272692224, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 97945387008, + "reserved_bytes_peak": 97945387008, + "segment_count_current": 464, + "segment_count_peak": 464, + "timestamp_utc": "2026-08-01T13:43:23.288432Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5997627904, + "inactive_split_bytes_current_delta": 855638016, + "inactive_split_bytes_peak": 11371053568, + "inactive_split_bytes_peak_delta": 2098361344, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 109108527104, + "reserved_bytes_current_delta": 11163140096, + "reserved_bytes_peak": 109108527104, + "reserved_bytes_peak_delta": 11163140096, + "segment_count_current": 516, + "segment_count_current_delta": 52, + "segment_count_peak": 516, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T134324308950Z.pickle", + "timestamp_utc": "2026-08-01T13:43:24.428034Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T134324308950Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T134324308950Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..281fa07bd751268f856a3c75a65b316990e6df20 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T134324308950Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c65f0f51bc8668cbcb4983fc50a7106ee8f2947bf2e8e5c1e33c28ae9b700c59 +size 6076607 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T134325474512Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T134325474512Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f32e18c2f84df9b45e9a0fa464eb74c9eff305d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T134325474512Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105154407936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105154407936, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474150400, + "device": 7, + "inactive_split_bytes_current": 5880187392, + "inactive_split_bytes_peak": 11535746560, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:25.474108Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104474150400, + "allocated_bytes_peak_delta": 8957440, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105154407936, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104465192960, + "device": 7, + "inactive_split_bytes_current": 5997627904, + "inactive_split_bytes_peak": 11371053568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 109108527104, + "reserved_bytes_peak": 109108527104, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T13:43:24.444473Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5880187392, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 11535746560, + "inactive_split_bytes_peak_delta": 164692992, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 110285029376, + "reserved_bytes_current_delta": 1176502272, + "reserved_bytes_peak": 110285029376, + "reserved_bytes_peak_delta": 1176502272, + "segment_count_current": 519, + "segment_count_current_delta": 3, + "segment_count_peak": 519, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T134325474512Z.pickle", + "timestamp_utc": "2026-08-01T13:43:25.586917Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T134325474512Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T134325474512Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0c6dabedd98b45059a139b9893f9de8f7674ea26 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T134325474512Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5bf36aa5d3eaeb10e1808a9ae7db4583e38480b95d9a81cada291653eb7c8b9 +size 6614305 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T134326899513Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T134326899513Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac7e8893d6953b7aede32b94d56b61290e6afb93 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T134326899513Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105154407936, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105154407936, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474150400, + "device": 7, + "inactive_split_bytes_current": 5464951296, + "inactive_split_bytes_peak": 11535746560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:26.899103Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104474150400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105154407936, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474150400, + "device": 7, + "inactive_split_bytes_current": 5880187392, + "inactive_split_bytes_peak": 11535746560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:25.602965Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5464951296, + "inactive_split_bytes_current_delta": -415236096, + "inactive_split_bytes_peak": 11535746560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 110285029376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110285029376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 519, + "segment_count_current_delta": 0, + "segment_count_peak": 519, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T134326899513Z.pickle", + "timestamp_utc": "2026-08-01T13:43:27.013301Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T134326899513Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T134326899513Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ee20f679025a2a6254f862b8551c3cc65c33d69 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T134326899513Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2ccf1b65a499832693695e08edc2376aa82bb359915b2241378027c6b2d8039 +size 6613055 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T134328352834Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T134328352834Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d00e687647bcaadcb8a8b3a87e13f916aa22b8f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T134328352834Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105165754880, + "active_bytes_peak_delta": 11346944, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105165754880, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474236416, + "device": 7, + "inactive_split_bytes_current": 5825661440, + "inactive_split_bytes_peak": 11535746560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:28.352417Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104474236416, + "allocated_bytes_peak_delta": 86016, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105154407936, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474150400, + "device": 7, + "inactive_split_bytes_current": 5464951296, + "inactive_split_bytes_peak": 11535746560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:27.029279Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5825661440, + "inactive_split_bytes_current_delta": 360710144, + "inactive_split_bytes_peak": 11535746560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 110285029376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110285029376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 519, + "segment_count_current_delta": 0, + "segment_count_peak": 519, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T134328352834Z.pickle", + "timestamp_utc": "2026-08-01T13:43:28.463857Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T134328352834Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T134328352834Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c7b00db01e372646e2dc1b033f70253b817979b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T134328352834Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca64db2b91d73a4d4ec34d9426c94fff9c677f0cd0d078d36df8993d0ac88a8a +size 6618856 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T134329515105Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T134329515105Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e3ec09153ae1e2f7570033ff333583a668ec3fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T134329515105Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105165754880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105165754880, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474236416, + "device": 7, + "inactive_split_bytes_current": 5733386752, + "inactive_split_bytes_peak": 11535746560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:29.514713Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104474236416, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105165754880, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474236416, + "device": 7, + "inactive_split_bytes_current": 5825661440, + "inactive_split_bytes_peak": 11535746560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:28.479616Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5733386752, + "inactive_split_bytes_current_delta": -92274688, + "inactive_split_bytes_peak": 11535746560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 110285029376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110285029376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 519, + "segment_count_current_delta": 0, + "segment_count_peak": 519, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T134329515105Z.pickle", + "timestamp_utc": "2026-08-01T13:43:29.624979Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T134329515105Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T134329515105Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e42cfc85ff7faa15523822b01879d0052bac8f2c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T134329515105Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d34df08a9e0010a79e794c7416b2f757d11e08b16dd896a52ab3f8e36bac658f +size 6618750 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T134330694850Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T134330694850Z.json new file mode 100644 index 0000000000000000000000000000000000000000..caabd2d146d3bb323c4bc6ab60e45bcc81805b9e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T134330694850Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105165754880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105165754880, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474236416, + "device": 7, + "inactive_split_bytes_current": 5351705088, + "inactive_split_bytes_peak": 11535746560, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:30.694437Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104474236416, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105165754880, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474236416, + "device": 7, + "inactive_split_bytes_current": 5733386752, + "inactive_split_bytes_peak": 11535746560, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:29.640759Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5351705088, + "inactive_split_bytes_current_delta": -381681664, + "inactive_split_bytes_peak": 11535746560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 110285029376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110285029376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 519, + "segment_count_current_delta": 0, + "segment_count_peak": 519, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T134330694850Z.pickle", + "timestamp_utc": "2026-08-01T13:43:30.806120Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T134330694850Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T134330694850Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ec4198c073671e00ad4b235b3c850f6a7deb4979 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T134330694850Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3c5a955d3e89670ca3521548799d57aedd3c2f5fecc743797862ff318edf8aa +size 6618639 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T134331996766Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T134331996766Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7b754729b507cd9793ca26c62ba3202c8cd36b4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T134331996766Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105165754880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105165754880, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474498560, + "device": 7, + "inactive_split_bytes_current": 5555128832, + "inactive_split_bytes_peak": 11535746560, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:31.996355Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104474498560, + "allocated_bytes_peak_delta": 262144, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105165754880, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474236416, + "device": 7, + "inactive_split_bytes_current": 5351705088, + "inactive_split_bytes_peak": 11535746560, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:30.821786Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5555128832, + "inactive_split_bytes_current_delta": 203423744, + "inactive_split_bytes_peak": 11535746560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 110285029376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110285029376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 519, + "segment_count_current_delta": 0, + "segment_count_peak": 519, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T134331996766Z.pickle", + "timestamp_utc": "2026-08-01T13:43:32.107511Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T134331996766Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T134331996766Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cdcba6ba700509559be5f046b634a1145e24876f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T134331996766Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfd52daadc038c5b92e245c161cea08f46186acc4be307e720a0b90e6a1d1334 +size 6618621 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T134333161174Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T134333161174Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1228add0e0d2167d65d62180ca34b91a74b39425 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T134333161174Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105165754880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105165754880, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474498560, + "device": 7, + "inactive_split_bytes_current": 5848730112, + "inactive_split_bytes_peak": 11535746560, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:33.160769Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 104474498560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105165754880, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474498560, + "device": 7, + "inactive_split_bytes_current": 5555128832, + "inactive_split_bytes_peak": 11535746560, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:32.123167Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5848730112, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 11535746560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 110285029376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110285029376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 519, + "segment_count_current_delta": 0, + "segment_count_peak": 519, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T134333161174Z.pickle", + "timestamp_utc": "2026-08-01T13:43:33.272595Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T134333161174Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T134333161174Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6d992a2edae075f08fbadfca023dcfc61fcb198a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T134333161174Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81216fbb8498103cc16b4a14dab147e4a4a8e4b0c9174c4ddc431e31f497ff2e +size 6612567 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T134426036531Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T134426036531Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ddd965ce01a61a95d0bf2ef5af612cf2e2454a29 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T134426036531Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165762048, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105165754880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165762048, + "active_bytes_peak": 105165754880, + "allocated_bytes_current": 61165762048, + "allocated_bytes_peak": 104474498560, + "device": 7, + "inactive_split_bytes_current": 5433494016, + "inactive_split_bytes_peak": 11535746560, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:44:26.036049Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165762048, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 104474498560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105165754880, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 104474498560, + "device": 7, + "inactive_split_bytes_current": 5848730112, + "inactive_split_bytes_peak": 11535746560, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 110285029376, + "reserved_bytes_peak": 110285029376, + "segment_count_current": 519, + "segment_count_peak": 519, + "timestamp_utc": "2026-08-01T13:43:33.290032Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5433494016, + "inactive_split_bytes_current_delta": -415236096, + "inactive_split_bytes_peak": 11535746560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 110285029376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110285029376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 519, + "segment_count_current_delta": 0, + "segment_count_peak": 519, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T134426036531Z.pickle", + "timestamp_utc": "2026-08-01T13:44:26.239503Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T134426036531Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T134426036531Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b7c80dcd45ef5ad533f86c9fd8f1e024438a86b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T134426036531Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1aa771f7e81ff2ef4dba810f6a8d90d4340eb34598a87ad83392f7b4e00e82bc +size 9724771 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..a4022a8f5d1e7f8085f9d3eedacd8a9762c1b4b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 2048 +capacity: 1.50 +expert_dispatch_overlap: off +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6032 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 226 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T134709572285Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T134709572285Z.json new file mode 100644 index 0000000000000000000000000000000000000000..09e1a5395b90119179f33ea19d55415e1fe55031 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T134709572285Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854876160, + "active_bytes_current_delta": 31623420928, + "active_bytes_peak": 82175548928, + "active_bytes_peak_delta": 46709212672, + "after": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 82175548928, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 82175548928, + "device": 0, + "inactive_split_bytes_current": 5085826048, + "inactive_split_bytes_peak": 7222522880, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 97469333504, + "reserved_bytes_peak": 97469333504, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:47:09.571885Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854876160, + "allocated_bytes_current_delta": 31623420928, + "allocated_bytes_peak": 82175548928, + "allocated_bytes_peak_delta": 46709212672, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:46:01.509005Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5085826048, + "inactive_split_bytes_current_delta": 4950909952, + "inactive_split_bytes_peak": 7222522880, + "inactive_split_bytes_peak_delta": 7061891072, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 97469333504, + "reserved_bytes_current_delta": 58223230976, + "reserved_bytes_peak": 97469333504, + "reserved_bytes_peak_delta": 58223230976, + "segment_count_current": 490, + "segment_count_current_delta": 418, + "segment_count_peak": 490, + "segment_count_peak_delta": 418, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T134709572285Z.pickle", + "timestamp_utc": "2026-08-01T13:47:09.679557Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T134709572285Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T134709572285Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8090fdf54403f6e08c253a2923f6e33ab4c49698 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T134709572285Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef5de1abf4f8a2d252a542808d52108d282c39bfb9a6fe0790d55abcfab0b53d +size 5378756 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T134710616448Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T134710616448Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1b5959412098b55687947b24597ad9888e770d39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T134710616448Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854876160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105806178304, + "active_bytes_peak_delta": 23630629376, + "after": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105806178304, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105806178304, + "device": 0, + "inactive_split_bytes_current": 4804807680, + "inactive_split_bytes_peak": 7222522880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107032346624, + "reserved_bytes_peak": 107032346624, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T13:47:10.616043Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854876160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105806178304, + "allocated_bytes_peak_delta": 23630629376, + "before": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 82175548928, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 82175548928, + "device": 0, + "inactive_split_bytes_current": 5085826048, + "inactive_split_bytes_peak": 7222522880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 97469333504, + "reserved_bytes_peak": 97469333504, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T13:47:09.695779Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4804807680, + "inactive_split_bytes_current_delta": -281018368, + "inactive_split_bytes_peak": 7222522880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107032346624, + "reserved_bytes_current_delta": 9563013120, + "reserved_bytes_peak": 107032346624, + "reserved_bytes_peak_delta": 9563013120, + "segment_count_current": 549, + "segment_count_current_delta": 59, + "segment_count_peak": 549, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T134710616448Z.pickle", + "timestamp_utc": "2026-08-01T13:47:10.726138Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T134710616448Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T134710616448Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b591d6e91e05ee220dabb175a8e0d73acd6e7f96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T134710616448Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ba09846b55145ced5ab270fa794c81ffc7b4e52fa8e55bdddeb0aca9d1b6d7e +size 5603464 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T134711600175Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T134711600175Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e576ed7c10d2dd41c512609f93ab2f32a8d07317 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T134711600175Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854876160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105809313792, + "active_bytes_peak_delta": 3135488, + "after": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105809313792, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105809313792, + "device": 0, + "inactive_split_bytes_current": 5329095680, + "inactive_split_bytes_peak": 7222522880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107032346624, + "reserved_bytes_peak": 107032346624, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T13:47:11.599785Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854876160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105809313792, + "allocated_bytes_peak_delta": 3135488, + "before": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105806178304, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105806178304, + "device": 0, + "inactive_split_bytes_current": 4804807680, + "inactive_split_bytes_peak": 7222522880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107032346624, + "reserved_bytes_peak": 107032346624, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T13:47:10.738213Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5329095680, + "inactive_split_bytes_current_delta": 524288000, + "inactive_split_bytes_peak": 7222522880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107032346624, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107032346624, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 549, + "segment_count_current_delta": 0, + "segment_count_peak": 549, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T134711600175Z.pickle", + "timestamp_utc": "2026-08-01T13:47:11.695454Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T134711600175Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T134711600175Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d4ff6af5f510aaf2a02a4a9e349791cae9341636 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T134711600175Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86c1675136532638f913230467ae6d05974c5f63a714e1caa8d49587e1f6e637 +size 5604139 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T134712614677Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T134712614677Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a0130ae0dd439b3a0634f487f5a4b60f37ea46ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T134712614677Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854876160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105809313792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105809313792, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105809313792, + "device": 0, + "inactive_split_bytes_current": 5480090624, + "inactive_split_bytes_peak": 7222522880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107502108672, + "reserved_bytes_peak": 107502108672, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T13:47:12.614293Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854876160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105809313792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105809313792, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105809313792, + "device": 0, + "inactive_split_bytes_current": 5329095680, + "inactive_split_bytes_peak": 7222522880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107032346624, + "reserved_bytes_peak": 107032346624, + "segment_count_current": 549, + "segment_count_peak": 549, + "timestamp_utc": "2026-08-01T13:47:11.706566Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5480090624, + "inactive_split_bytes_current_delta": 150994944, + "inactive_split_bytes_peak": 7222522880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107502108672, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107502108672, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 550, + "segment_count_current_delta": 1, + "segment_count_peak": 550, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T134712614677Z.pickle", + "timestamp_utc": "2026-08-01T13:47:12.709495Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T134712614677Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T134712614677Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7d3431c4f2f7f6f17c21b32e9d3784a532ab441c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T134712614677Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77efcf2689c54fbfb7beec9c6e34c68fe89b1e42b1b188c534a763fd526c8d2a +size 5602710 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T134714068807Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T134714068807Z.json new file mode 100644 index 0000000000000000000000000000000000000000..91780038e1c8a9e4e6caa59816e00b235b2b8995 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T134714068807Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854876160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813125632, + "active_bytes_peak_delta": 3811840, + "after": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105813125632, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105813125632, + "device": 0, + "inactive_split_bytes_current": 5685611520, + "inactive_split_bytes_peak": 7222522880, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107502108672, + "reserved_bytes_peak": 107502108672, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T13:47:14.068388Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854876160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813125632, + "allocated_bytes_peak_delta": 3811840, + "before": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105809313792, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105809313792, + "device": 0, + "inactive_split_bytes_current": 5480090624, + "inactive_split_bytes_peak": 7222522880, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107502108672, + "reserved_bytes_peak": 107502108672, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T13:47:12.720426Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5685611520, + "inactive_split_bytes_current_delta": 205520896, + "inactive_split_bytes_peak": 7222522880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107502108672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107502108672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 550, + "segment_count_current_delta": 0, + "segment_count_peak": 550, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T134714068807Z.pickle", + "timestamp_utc": "2026-08-01T13:47:14.170845Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T134714068807Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T134714068807Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ee58fbcbb15fef529b3f42027391c77839863b14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T134714068807Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3ef93f820aba73180bef648ab21401ddd97f911bf410d1fde37a54194c818cd +size 5601547 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T134715069771Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T134715069771Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1dc06851ea83dc3bdc9d80fb861c8397dddc625f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T134715069771Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854876160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813125632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105813125632, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105813125632, + "device": 0, + "inactive_split_bytes_current": 5503159296, + "inactive_split_bytes_peak": 7222522880, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107502108672, + "reserved_bytes_peak": 107502108672, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T13:47:15.069390Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854876160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813125632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105813125632, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105813125632, + "device": 0, + "inactive_split_bytes_current": 5685611520, + "inactive_split_bytes_peak": 7222522880, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107502108672, + "reserved_bytes_peak": 107502108672, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T13:47:14.182645Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5503159296, + "inactive_split_bytes_current_delta": -182452224, + "inactive_split_bytes_peak": 7222522880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107502108672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107502108672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 550, + "segment_count_current_delta": 0, + "segment_count_peak": 550, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T134715069771Z.pickle", + "timestamp_utc": "2026-08-01T13:47:15.163250Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T134715069771Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T134715069771Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eab499fe9aaf1a0e7e62a09658b814da59671bbf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T134715069771Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe93c33e1800347b0d5b1d32051ba87f9efc3b8929280a02bbbb169ef643415c +size 5602294 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T134716067388Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T134716067388Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e05c96ce965edd796b56abbcd34999ff302e374 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T134716067388Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854876160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813125632, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105813125632, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105813125632, + "device": 0, + "inactive_split_bytes_current": 5184392192, + "inactive_split_bytes_peak": 7222522880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107502108672, + "reserved_bytes_peak": 107502108672, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T13:47:16.066982Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854876160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813125632, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105813125632, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105813125632, + "device": 0, + "inactive_split_bytes_current": 5503159296, + "inactive_split_bytes_peak": 7222522880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107502108672, + "reserved_bytes_peak": 107502108672, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T13:47:15.174156Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5184392192, + "inactive_split_bytes_current_delta": -318767104, + "inactive_split_bytes_peak": 7222522880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107502108672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107502108672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 550, + "segment_count_current_delta": 0, + "segment_count_peak": 550, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T134716067388Z.pickle", + "timestamp_utc": "2026-08-01T13:47:16.166475Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T134716067388Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T134716067388Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ae1aceff95f5937d5999b3fac13f771268ae731 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T134716067388Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90f9aa6a023ffd362f7254c689f8b60ac7004c1415348d9475e1f262e46187d2 +size 5602210 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T134717073879Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T134717073879Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f6d1aa5ea396a8fcd821413c85e5fca269a3e46 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T134717073879Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854876160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814012416, + "active_bytes_peak_delta": 886784, + "after": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105814012416, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105814012416, + "device": 0, + "inactive_split_bytes_current": 5708680192, + "inactive_split_bytes_peak": 7251311616, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107504205824, + "reserved_bytes_peak": 107504205824, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-01T13:47:17.073478Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854876160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814012416, + "allocated_bytes_peak_delta": 886784, + "before": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105813125632, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105813125632, + "device": 0, + "inactive_split_bytes_current": 5184392192, + "inactive_split_bytes_peak": 7222522880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107502108672, + "reserved_bytes_peak": 107502108672, + "segment_count_current": 550, + "segment_count_peak": 550, + "timestamp_utc": "2026-08-01T13:47:16.181266Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5708680192, + "inactive_split_bytes_current_delta": 524288000, + "inactive_split_bytes_peak": 7251311616, + "inactive_split_bytes_peak_delta": 28788736, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107504205824, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 107504205824, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 551, + "segment_count_current_delta": 1, + "segment_count_peak": 551, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T134717073879Z.pickle", + "timestamp_utc": "2026-08-01T13:47:17.169724Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T134717073879Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T134717073879Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8767c0ed71704a37bd0e07b08ce91b72cb3cdbfe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T134717073879Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34a81d23dad772de4bb5c2520471a1445b78edc06b5ef78d344f5f091ffef8fc +size 5602134 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T134718069013Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T134718069013Z.json new file mode 100644 index 0000000000000000000000000000000000000000..134833af40e4c36dbc3cbcb5439d4d1cb55ede5d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T134718069013Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854876160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814012416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105814012416, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105814012416, + "device": 0, + "inactive_split_bytes_current": 5536713728, + "inactive_split_bytes_peak": 7251311616, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107504205824, + "reserved_bytes_peak": 107504205824, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-01T13:47:18.068620Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854876160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814012416, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105814012416, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105814012416, + "device": 0, + "inactive_split_bytes_current": 5708680192, + "inactive_split_bytes_peak": 7251311616, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107504205824, + "reserved_bytes_peak": 107504205824, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-01T13:47:17.180654Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5536713728, + "inactive_split_bytes_current_delta": -171966464, + "inactive_split_bytes_peak": 7251311616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107504205824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107504205824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 551, + "segment_count_current_delta": 0, + "segment_count_peak": 551, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T134718069013Z.pickle", + "timestamp_utc": "2026-08-01T13:47:18.160426Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T134718069013Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T134718069013Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b38c21a41d78c911455201decfbc1440f6ac920c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T134718069013Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b19fb143cf0176099764eb90c151e5add7c135f77a178089c0bae2000fad53e +size 5602595 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T134814745498Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T134814745498Z.json new file mode 100644 index 0000000000000000000000000000000000000000..348cb701b1199172bf8daa85e6d1c6dfcf0b1e12 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T134814745498Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804233216, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105814012416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66804233216, + "active_bytes_peak": 105814012416, + "allocated_bytes_current": 66804233216, + "allocated_bytes_peak": 105814012416, + "device": 0, + "inactive_split_bytes_current": 3469233152, + "inactive_split_bytes_peak": 7347513344, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107504205824, + "reserved_bytes_peak": 107504205824, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-01T13:48:14.745039Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804233216, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105814012416, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854876160, + "active_bytes_peak": 105814012416, + "allocated_bytes_current": 66854876160, + "allocated_bytes_peak": 105814012416, + "device": 0, + "inactive_split_bytes_current": 5536713728, + "inactive_split_bytes_peak": 7251311616, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107504205824, + "reserved_bytes_peak": 107504205824, + "segment_count_current": 551, + "segment_count_peak": 551, + "timestamp_utc": "2026-08-01T13:47:18.171113Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 3469233152, + "inactive_split_bytes_current_delta": -2067480576, + "inactive_split_bytes_peak": 7347513344, + "inactive_split_bytes_peak_delta": 96201728, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107504205824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107504205824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 551, + "segment_count_current_delta": 0, + "segment_count_peak": 551, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T134814745498Z.pickle", + "timestamp_utc": "2026-08-01T13:48:14.946071Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T134814745498Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T134814745498Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1eb16d79eda98eb344ff227565424e4527627ea1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T134814745498Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d109caa49e3b93fcac28e256631ac84431e30cd5a456501dbbaa824c9d54b2a +size 9349518 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T134709572423Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T134709572423Z.json new file mode 100644 index 0000000000000000000000000000000000000000..95474d82be93327dfba0ef5bf5aef0659ada26a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T134709572423Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 31622988800, + "active_bytes_peak": 82174059520, + "active_bytes_peak_delta": 46707723264, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 82174059520, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 82174059520, + "device": 1, + "inactive_split_bytes_current": 5279196160, + "inactive_split_bytes_peak": 7083843584, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:47:09.571819Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 31622988800, + "allocated_bytes_peak": 82174059520, + "allocated_bytes_peak_delta": 46707723264, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:46:01.509445Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5279196160, + "inactive_split_bytes_current_delta": 5146377216, + "inactive_split_bytes_peak": 7083843584, + "inactive_split_bytes_peak_delta": 6923457536, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 491, + "segment_count_current_delta": 420, + "segment_count_peak": 491, + "segment_count_peak_delta": 420, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T134709572423Z.pickle", + "timestamp_utc": "2026-08-01T13:47:09.681500Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T134709572423Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T134709572423Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c559a26683bea814eb37bf9a7657dcbfbb0f21f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T134709572423Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c53b7c30a24429c4ebca47633365f59c03fa038d8903e243db6c7a01d764b6b1 +size 5370443 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T134710616510Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T134710616510Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5077d06fb1027a61cc83c48154b9fc3acbe4f93b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T134710616510Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813069824, + "active_bytes_peak_delta": 23639010304, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813069824, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813069824, + "device": 1, + "inactive_split_bytes_current": 5342110720, + "inactive_split_bytes_peak": 7083843584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:10.616089Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813069824, + "allocated_bytes_peak_delta": 23639010304, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 82174059520, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 82174059520, + "device": 1, + "inactive_split_bytes_current": 5279196160, + "inactive_split_bytes_peak": 7083843584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:47:09.698772Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5342110720, + "inactive_split_bytes_current_delta": 62914560, + "inactive_split_bytes_peak": 7083843584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 9521070080, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 9521070080, + "segment_count_current": 546, + "segment_count_current_delta": 55, + "segment_count_peak": 546, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T134710616510Z.pickle", + "timestamp_utc": "2026-08-01T13:47:10.727373Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T134710616510Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T134710616510Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2418ebb14b97014a0943e27039cca19339997d99 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T134710616510Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:143077fe6d47c484cecc4e8811fb5cac9606a46d970f54bec3ac7978066fa5c5 +size 5559714 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T134711600221Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T134711600221Z.json new file mode 100644 index 0000000000000000000000000000000000000000..acfc135530287f47c81cef8d5ddef499f29d6f92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T134711600221Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813069824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813069824, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813069824, + "device": 1, + "inactive_split_bytes_current": 5222573056, + "inactive_split_bytes_peak": 7083843584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:11.599820Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813069824, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813069824, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813069824, + "device": 1, + "inactive_split_bytes_current": 5342110720, + "inactive_split_bytes_peak": 7083843584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:10.740917Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5222573056, + "inactive_split_bytes_current_delta": -119537664, + "inactive_split_bytes_peak": 7083843584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T134711600221Z.pickle", + "timestamp_utc": "2026-08-01T13:47:11.702752Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T134711600221Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T134711600221Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b9fa099b5aa784bff09bcf3142cc056e4f480a9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T134711600221Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d81aa237a1017f13abc41f26d57aa39d6565288ef8dc94bdc6a846c4024807f +size 5568293 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T134712614778Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T134712614778Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0422594b852a028dcf0d60efee627ab2cd5d9116 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T134712614778Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813966848, + "active_bytes_peak_delta": 897024, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813966848, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813966848, + "device": 1, + "inactive_split_bytes_current": 6101279744, + "inactive_split_bytes_peak": 7160159232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:12.614388Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813966848, + "allocated_bytes_peak_delta": 897024, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813069824, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813069824, + "device": 1, + "inactive_split_bytes_current": 5222573056, + "inactive_split_bytes_peak": 7083843584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:11.715798Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6101279744, + "inactive_split_bytes_current_delta": 878706688, + "inactive_split_bytes_peak": 7160159232, + "inactive_split_bytes_peak_delta": 76315648, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T134712614778Z.pickle", + "timestamp_utc": "2026-08-01T13:47:12.716892Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T134712614778Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T134712614778Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d8e39a75a4324d305e03a70767a31179b701d0ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T134712614778Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aad070f9b71a0bd31d0040072476245134cfbd2a949298a30bbb73723cfd6bc +size 5567629 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T134714068833Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T134714068833Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1692ec1c6efb5e6036b80349de7d02416efd67c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T134714068833Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815712256, + "active_bytes_peak_delta": 1745408, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105815712256, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105815712256, + "device": 1, + "inactive_split_bytes_current": 5929313280, + "inactive_split_bytes_peak": 7236836352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:14.068427Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815712256, + "allocated_bytes_peak_delta": 1745408, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813966848, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813966848, + "device": 1, + "inactive_split_bytes_current": 6101279744, + "inactive_split_bytes_peak": 7160159232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:12.729848Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5929313280, + "inactive_split_bytes_current_delta": -171966464, + "inactive_split_bytes_peak": 7236836352, + "inactive_split_bytes_peak_delta": 76677120, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T134714068833Z.pickle", + "timestamp_utc": "2026-08-01T13:47:14.173817Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T134714068833Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T134714068833Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1da956c1666569898f3fb16798e78791892b7d9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T134714068833Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4781d30cc87c553f8559349d86f3558cd9f23bbd8bf3296397dab70c8d46de70 +size 5566553 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T134715069899Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T134715069899Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ae653294e2ab22ee9d298a00f2246518ebbed4c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T134715069899Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815712256, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105815712256, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105815712256, + "device": 1, + "inactive_split_bytes_current": 5249836032, + "inactive_split_bytes_peak": 7236836352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:15.069503Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815712256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105815712256, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105815712256, + "device": 1, + "inactive_split_bytes_current": 5929313280, + "inactive_split_bytes_peak": 7236836352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:14.187028Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5249836032, + "inactive_split_bytes_current_delta": -679477248, + "inactive_split_bytes_peak": 7236836352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T134715069899Z.pickle", + "timestamp_utc": "2026-08-01T13:47:15.174205Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T134715069899Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T134715069899Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..af40a518f4696885cba055ca623c4d7cc216b096 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T134715069899Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:260d2a3ca124f1a9be208ae5c9743373a3b89aaac98ca499aacc0767ffe24dd3 +size 5567625 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T134716067443Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T134716067443Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d0a1157d9a3a8be8e96e3df3625fb05e99665d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T134716067443Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 2580480, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 1, + "inactive_split_bytes_current": 6313092096, + "inactive_split_bytes_peak": 7236836352, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:16.067019Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 2580480, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105815712256, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105815712256, + "device": 1, + "inactive_split_bytes_current": 5249836032, + "inactive_split_bytes_peak": 7236836352, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:15.189469Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6313092096, + "inactive_split_bytes_current_delta": 1063256064, + "inactive_split_bytes_peak": 7236836352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T134716067443Z.pickle", + "timestamp_utc": "2026-08-01T13:47:16.171603Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T134716067443Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T134716067443Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..adb3107d44f9a93545c3cd860968cbc9e476b809 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T134716067443Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3dae6abd5c954f92e14a8127c7017af64f59a0dc635f51724fac57c9a67f483 +size 5566299 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T134717073901Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T134717073901Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a13da93e2f1f225dc6ab5c1579340be59c555d9e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T134717073901Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 1, + "inactive_split_bytes_current": 5931410432, + "inactive_split_bytes_peak": 7238933504, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:17.073511Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 1, + "inactive_split_bytes_current": 6313092096, + "inactive_split_bytes_peak": 7236836352, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:16.185315Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5931410432, + "inactive_split_bytes_current_delta": -381681664, + "inactive_split_bytes_peak": 7238933504, + "inactive_split_bytes_peak_delta": 2097152, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T134717073901Z.pickle", + "timestamp_utc": "2026-08-01T13:47:17.181723Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T134717073901Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T134717073901Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6130077e6b5610a1d6ab984b5919878dd82477f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T134717073901Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fff36a20edb80c624c7cb3dcb6a91384bc424c483cd8beeb55214bc3dbb154ed +size 5584232 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T134718069070Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T134718069070Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34fa968890af428e6047e596f918f5b696d58906 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T134718069070Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 1, + "inactive_split_bytes_current": 5256127488, + "inactive_split_bytes_peak": 7238933504, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:18.068666Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 1, + "inactive_split_bytes_current": 5931410432, + "inactive_split_bytes_peak": 7238933504, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:17.195215Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5256127488, + "inactive_split_bytes_current_delta": -675282944, + "inactive_split_bytes_peak": 7238933504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T134718069070Z.pickle", + "timestamp_utc": "2026-08-01T13:47:18.167410Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T134718069070Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T134718069070Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b839b5aa939d82964de5b7ef072346a87d9e378e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T134718069070Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:699ff6fb0949e49a35bf3cf38cbab581edbb64a08c809485c0a6f249455fad6a +size 5600785 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T134814745533Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T134814745533Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f43d748e130bec8e39c8513ae3e051eea861a418 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T134814745533Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803801088, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803801088, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66803801088, + "allocated_bytes_peak": 105818292736, + "device": 1, + "inactive_split_bytes_current": 3603883008, + "inactive_split_bytes_peak": 7238933504, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:48:14.745054Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803801088, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 1, + "inactive_split_bytes_current": 5256127488, + "inactive_split_bytes_peak": 7238933504, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:18.180392Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 3603883008, + "inactive_split_bytes_current_delta": -1652244480, + "inactive_split_bytes_peak": 7238933504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T134814745533Z.pickle", + "timestamp_utc": "2026-08-01T13:48:14.943563Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T134814745533Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T134814745533Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c14696709dac2c506d7608c67e04caf9ffbe395f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T134814745533Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f370ceac8af39c69bb7752cee1e7fa5f247cc09049b6b073874f3e3c973d804 +size 9250024 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T134709572296Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T134709572296Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b099f35134636c0753beef03954d7b9ef73f9e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T134709572296Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 31622988800, + "active_bytes_peak": 82174059520, + "active_bytes_peak_delta": 46707723264, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 82174059520, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 82174059520, + "device": 2, + "inactive_split_bytes_current": 5275001856, + "inactive_split_bytes_peak": 7081828352, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:47:09.571715Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 31622988800, + "allocated_bytes_peak": 82174059520, + "allocated_bytes_peak_delta": 46707723264, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:46:01.509569Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5275001856, + "inactive_split_bytes_current_delta": 5142182912, + "inactive_split_bytes_peak": 7081828352, + "inactive_split_bytes_peak_delta": 6921442304, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 491, + "segment_count_current_delta": 420, + "segment_count_peak": 491, + "segment_count_peak_delta": 420, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T134709572296Z.pickle", + "timestamp_utc": "2026-08-01T13:47:09.680592Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T134709572296Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T134709572296Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a995f6460cff5eb51e16938cbd09dc6514b6db3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T134709572296Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02ecdef40d412504e86891e9501159740412357db3d2c112abd352beb0c7355b +size 5369882 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T134710616456Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T134710616456Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bae540d7d043a571f0c839f61166250233435f30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T134710616456Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815472640, + "active_bytes_peak_delta": 23641413120, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105815472640, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105815472640, + "device": 2, + "inactive_split_bytes_current": 5275001856, + "inactive_split_bytes_peak": 7081828352, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:10.616038Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815472640, + "allocated_bytes_peak_delta": 23641413120, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 82174059520, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 82174059520, + "device": 2, + "inactive_split_bytes_current": 5275001856, + "inactive_split_bytes_peak": 7081828352, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:47:09.697898Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5275001856, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7081828352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 9521070080, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 9521070080, + "segment_count_current": 546, + "segment_count_current_delta": 55, + "segment_count_peak": 546, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T134710616456Z.pickle", + "timestamp_utc": "2026-08-01T13:47:10.726791Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T134710616456Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T134710616456Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..67b6963c0795e3a366107bc7763c154bb92c409b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T134710616456Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1333bcce65a555b11292b2be58744a3981a5f4a522bb3b87e67839ce9371cf91 +size 5566629 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T134711600109Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T134711600109Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4faeab4fc3ee5837f03504d0545fe51f068bb978 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T134711600109Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815472640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105815472640, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105815472640, + "device": 2, + "inactive_split_bytes_current": 5576991744, + "inactive_split_bytes_peak": 7081828352, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:11.599715Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815472640, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105815472640, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105815472640, + "device": 2, + "inactive_split_bytes_current": 5275001856, + "inactive_split_bytes_peak": 7081828352, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:10.740127Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5576991744, + "inactive_split_bytes_current_delta": 301989888, + "inactive_split_bytes_peak": 7081828352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T134711600109Z.pickle", + "timestamp_utc": "2026-08-01T13:47:11.699988Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T134711600109Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T134711600109Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80cbc84584c676dd6b1b2515becaaf86dbf3e812 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T134711600109Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:831360a7e52c199084010b78c2a338e3fdeef769e69ae79f79dedd4d6706e955 +size 5567915 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T134712614628Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T134712614628Z.json new file mode 100644 index 0000000000000000000000000000000000000000..933aa4cb9205d91532e7837f48ec5975f5a4a351 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T134712614628Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105817305600, + "active_bytes_peak_delta": 1832960, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105817305600, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105817305600, + "device": 2, + "inactive_split_bytes_current": 5692335104, + "inactive_split_bytes_peak": 7081828352, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:12.614226Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105817305600, + "allocated_bytes_peak_delta": 1832960, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105815472640, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105815472640, + "device": 2, + "inactive_split_bytes_current": 5576991744, + "inactive_split_bytes_peak": 7081828352, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:11.712616Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5692335104, + "inactive_split_bytes_current_delta": 115343360, + "inactive_split_bytes_peak": 7081828352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T134712614628Z.pickle", + "timestamp_utc": "2026-08-01T13:47:12.712559Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T134712614628Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T134712614628Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0b54ce575aa2c3fabc3f3fa3006ee389561791ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T134712614628Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddf7c69e79b0b61ee1b99d61d0c0e7f676ff92fe9c499fb7e0a8c730c379b124 +size 5567816 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T134714068727Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T134714068727Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fbfd77d66fa8780c6b68dec2ec6d41c68f2fb766 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T134714068727Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 987136, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 2, + "inactive_split_bytes_current": 5279196160, + "inactive_split_bytes_peak": 7081828352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:14.068319Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 987136, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105817305600, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105817305600, + "device": 2, + "inactive_split_bytes_current": 5692335104, + "inactive_split_bytes_peak": 7081828352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:12.725029Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5279196160, + "inactive_split_bytes_current_delta": -413138944, + "inactive_split_bytes_peak": 7081828352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T134714068727Z.pickle", + "timestamp_utc": "2026-08-01T13:47:14.170939Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T134714068727Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T134714068727Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9246b3da275d88a10e3970c230db35fbed096f83 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T134714068727Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:201029851fba8a4830db4828def3bea3e51ba47c5b561d626972a2d62701d0b0 +size 5566167 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T134715069735Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T134715069735Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aaaf8cbd358a40f6833e3a1d52662a7aa73338ef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T134715069735Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 2, + "inactive_split_bytes_current": 5751055360, + "inactive_split_bytes_peak": 7081828352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:15.069344Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 2, + "inactive_split_bytes_current": 5279196160, + "inactive_split_bytes_peak": 7081828352, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:14.183907Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5751055360, + "inactive_split_bytes_current_delta": 471859200, + "inactive_split_bytes_peak": 7081828352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T134715069735Z.pickle", + "timestamp_utc": "2026-08-01T13:47:15.166688Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T134715069735Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T134715069735Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7dbdea1bedd2a607b665536b271eabc6e499dc8b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T134715069735Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a9d386b950b604fcdb76f1a75d62bd11edbcee7bcf2c282fdec161243b5dbf1 +size 5576028 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T134716067311Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T134716067311Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a4569dd93976f8b1443fa52088aa3a0b15220942 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T134716067311Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 2, + "inactive_split_bytes_current": 5457454080, + "inactive_split_bytes_peak": 7081828352, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:16.066908Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 2, + "inactive_split_bytes_current": 5751055360, + "inactive_split_bytes_peak": 7081828352, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:15.179046Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5457454080, + "inactive_split_bytes_current_delta": -293601280, + "inactive_split_bytes_peak": 7081828352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T134716067311Z.pickle", + "timestamp_utc": "2026-08-01T13:47:16.174275Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T134716067311Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T134716067311Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de720cf707b818ff73c519109f6964119705e3d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T134716067311Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:717b660fd963a2cca4bc4da046b964013045e2576f227af0880ec7d5d78a4a0b +size 5567260 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T134717073802Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T134717073802Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7cf1fa8b2bd0dcec5d7b69c423b0934e47b67288 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T134717073802Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 2, + "inactive_split_bytes_current": 5633614848, + "inactive_split_bytes_peak": 7263098880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:17.073409Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 2, + "inactive_split_bytes_current": 5457454080, + "inactive_split_bytes_peak": 7081828352, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:16.195295Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5633614848, + "inactive_split_bytes_current_delta": 176160768, + "inactive_split_bytes_peak": 7263098880, + "inactive_split_bytes_peak_delta": 181270528, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T134717073802Z.pickle", + "timestamp_utc": "2026-08-01T13:47:17.172470Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T134717073802Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T134717073802Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9201202d8a37f2badaa2fa4a173c44b9ddf6142d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T134717073802Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e30bee687f48cc72f794bb0731774c9c9efdcfe5f942de5f85f69525d4246bd +size 5566765 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T134718068975Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T134718068975Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a2f4d95af5dfb94eb45a903ac805b7386cb5e47 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T134718068975Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 2, + "inactive_split_bytes_current": 5929313280, + "inactive_split_bytes_peak": 7263098880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:18.068567Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 2, + "inactive_split_bytes_current": 5633614848, + "inactive_split_bytes_peak": 7263098880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:17.184711Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5929313280, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 7263098880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T134718068975Z.pickle", + "timestamp_utc": "2026-08-01T13:47:18.164495Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T134718068975Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T134718068975Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ecbdf5e34ab50a347fe9936e8d95ea6e7afe400 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T134718068975Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dd90ca91874941db57e7246c37c15b380ebdee3e90f79799ea20acf0782f47e +size 5566007 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T134814745529Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T134814745529Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a265866e5da1719a027e21b0f10cee902eb0c8f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T134814745529Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803801088, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803801088, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66803801088, + "allocated_bytes_peak": 105818292736, + "device": 2, + "inactive_split_bytes_current": 3603883008, + "inactive_split_bytes_peak": 7263098880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:48:14.745062Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803801088, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 2, + "inactive_split_bytes_current": 5929313280, + "inactive_split_bytes_peak": 7263098880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:18.176839Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 3603883008, + "inactive_split_bytes_current_delta": -2325430272, + "inactive_split_bytes_peak": 7263098880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T134814745529Z.pickle", + "timestamp_utc": "2026-08-01T13:48:14.941071Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T134814745529Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T134814745529Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a449684d75209459a5d547f679ad86031001d96f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T134814745529Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b2ba01c500bdc94f5f298be2d9e32b8b0be6e667f28f029fc8633aefd966808 +size 9207272 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T134709572288Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T134709572288Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c86a358bc98c9cf09a6cc6df912dc107ebf781fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T134709572288Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 31622988800, + "active_bytes_peak": 82174059520, + "active_bytes_peak_delta": 46707723264, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 82174059520, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 82174059520, + "device": 3, + "inactive_split_bytes_current": 5277099008, + "inactive_split_bytes_peak": 7081828352, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:47:09.571695Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 31622988800, + "allocated_bytes_peak": 82174059520, + "allocated_bytes_peak_delta": 46707723264, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:46:01.509477Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5277099008, + "inactive_split_bytes_current_delta": 5144280064, + "inactive_split_bytes_peak": 7081828352, + "inactive_split_bytes_peak_delta": 6921442304, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 491, + "segment_count_current_delta": 420, + "segment_count_peak": 491, + "segment_count_peak_delta": 420, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T134709572288Z.pickle", + "timestamp_utc": "2026-08-01T13:47:09.683309Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T134709572288Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T134709572288Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2dc46aef9c395602734dc053577813a0d2f55011 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T134709572288Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eac4df19346fbe7905eb6b90dfa03d5ebdce6efc32690e2a1ea58640d3a205cc +size 5370369 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T134710616393Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T134710616393Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e3a6c9e832b4d1da3a6c9f14914b5537f0800ddd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T134710616393Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813069824, + "active_bytes_peak_delta": 23639010304, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813069824, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813069824, + "device": 3, + "inactive_split_bytes_current": 5459551232, + "inactive_split_bytes_peak": 7081828352, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:10.615985Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813069824, + "allocated_bytes_peak_delta": 23639010304, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 82174059520, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 82174059520, + "device": 3, + "inactive_split_bytes_current": 5277099008, + "inactive_split_bytes_peak": 7081828352, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:47:09.702935Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5459551232, + "inactive_split_bytes_current_delta": 182452224, + "inactive_split_bytes_peak": 7081828352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 9521070080, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 9521070080, + "segment_count_current": 546, + "segment_count_current_delta": 55, + "segment_count_peak": 546, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T134710616393Z.pickle", + "timestamp_utc": "2026-08-01T13:47:10.730208Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T134710616393Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T134710616393Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f3a4d0ed9b5876e6e3d72d61b4285e94d69dce4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T134710616393Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5629cc6a9b41bffd980431d4aa866204a68676a1e705e0c074517589be5b58a1 +size 5563160 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T134711600112Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T134711600112Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e1b25c4ea42eab6ac30022264d8a04cf1118f07 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T134711600112Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814119424, + "active_bytes_peak_delta": 1049600, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105814119424, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105814119424, + "device": 3, + "inactive_split_bytes_current": 5635712000, + "inactive_split_bytes_peak": 7349992448, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:11.599724Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814119424, + "allocated_bytes_peak_delta": 1049600, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813069824, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813069824, + "device": 3, + "inactive_split_bytes_current": 5459551232, + "inactive_split_bytes_peak": 7081828352, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:10.745242Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5635712000, + "inactive_split_bytes_current_delta": 176160768, + "inactive_split_bytes_peak": 7349992448, + "inactive_split_bytes_peak_delta": 268164096, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T134711600112Z.pickle", + "timestamp_utc": "2026-08-01T13:47:11.711067Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T134711600112Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T134711600112Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..42959f8daa918e1c42fcebb82708d83170cedb0b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T134711600112Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f8dbfd19f485903dffb2594e1142dff372f9f1dc4bb485bdaa6f5d5850f490e +size 5592113 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T134712614635Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T134712614635Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bdebde3b410f9137d999f0dcc9ed13ed331aca65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T134712614635Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105816257024, + "active_bytes_peak_delta": 2137600, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 3, + "inactive_split_bytes_current": 5843330048, + "inactive_split_bytes_peak": 7373883392, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:12.614240Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105816257024, + "allocated_bytes_peak_delta": 2137600, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105814119424, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105814119424, + "device": 3, + "inactive_split_bytes_current": 5635712000, + "inactive_split_bytes_peak": 7349992448, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:11.728548Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5843330048, + "inactive_split_bytes_current_delta": 207618048, + "inactive_split_bytes_peak": 7373883392, + "inactive_split_bytes_peak_delta": 23890944, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T134712614635Z.pickle", + "timestamp_utc": "2026-08-01T13:47:12.716910Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T134712614635Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T134712614635Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6cc3fc7374c875a2dcc833438be57e375968f7cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T134712614635Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef02f4f122052f9e8fdc1ee478525a035c0caabbb2568e5fcebac8a17fc03986 +size 5568139 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T134714068792Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T134714068792Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d81b46db5d0098b471068a4e2bb2221a4f3f436d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T134714068792Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105816257024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 3, + "inactive_split_bytes_current": 5960770560, + "inactive_split_bytes_peak": 7373883392, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:14.068389Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105816257024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 3, + "inactive_split_bytes_current": 5843330048, + "inactive_split_bytes_peak": 7373883392, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:12.731513Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5960770560, + "inactive_split_bytes_current_delta": 117440512, + "inactive_split_bytes_peak": 7373883392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T134714068792Z.pickle", + "timestamp_utc": "2026-08-01T13:47:14.174843Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T134714068792Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T134714068792Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1352cfdcab35b574e861f491be9212afdc407ccb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T134714068792Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:007431da94ef41d51f39b9e44b18fe83f2922d8eaa5d428b910d67f35ce19b1e +size 5565399 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T134715069753Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T134715069753Z.json new file mode 100644 index 0000000000000000000000000000000000000000..73575fcf4da9e4bed7fb921ade75449fd59adc87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T134715069753Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105816257024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 3, + "inactive_split_bytes_current": 6044656640, + "inactive_split_bytes_peak": 7373883392, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:15.069346Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105816257024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 3, + "inactive_split_bytes_current": 5960770560, + "inactive_split_bytes_peak": 7373883392, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:14.189309Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6044656640, + "inactive_split_bytes_current_delta": 83886080, + "inactive_split_bytes_peak": 7373883392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T134715069753Z.pickle", + "timestamp_utc": "2026-08-01T13:47:15.171378Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T134715069753Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T134715069753Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a5d1e2f832aa614284bcf209e02180b5dd50905 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T134715069753Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d919a9afd1531ad8de1645430d1bbe933b7527bf575a5b4995306774a69b2030 +size 5599370 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T134716067302Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T134716067302Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e64cf24988f69d5e5f606a42b2bc89fb4c4825c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T134716067302Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105816257024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 3, + "inactive_split_bytes_current": 5373568000, + "inactive_split_bytes_peak": 7373883392, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:16.066887Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105816257024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 3, + "inactive_split_bytes_current": 6044656640, + "inactive_split_bytes_peak": 7373883392, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:15.185852Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5373568000, + "inactive_split_bytes_current_delta": -671088640, + "inactive_split_bytes_peak": 7373883392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T134716067302Z.pickle", + "timestamp_utc": "2026-08-01T13:47:16.171524Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T134716067302Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T134716067302Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e504d3cbe6191800f8a0ec1be9ac7e3a40ed7a31 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T134716067302Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:466deac5b6cbd42548a9c8371399680c3a711b55b1b63b26683212b8cc076911 +size 5566748 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T134717073964Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T134717073964Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d3ac9d4237dabddb614ce8a6e13c1fd6f5743f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T134717073964Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 2035712, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 3, + "inactive_split_bytes_current": 6313092096, + "inactive_split_bytes_peak": 7373883392, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:17.073555Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 2035712, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 3, + "inactive_split_bytes_current": 5373568000, + "inactive_split_bytes_peak": 7373883392, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:16.186418Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6313092096, + "inactive_split_bytes_current_delta": 939524096, + "inactive_split_bytes_peak": 7373883392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T134717073964Z.pickle", + "timestamp_utc": "2026-08-01T13:47:17.175771Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T134717073964Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T134717073964Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c216dcf0ccb253ba3317d8b1614db25ab7f9c37e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T134717073964Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8712accbb7dc6b0b07aaa181c79481df285a67947afd32d825113e29953c119f +size 5565880 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T134718068981Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T134718068981Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1604a7ea3a001131f2deeab9f93228e111e74334 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T134718068981Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 3, + "inactive_split_bytes_current": 5925118976, + "inactive_split_bytes_peak": 7373883392, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:18.068570Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 3, + "inactive_split_bytes_current": 6313092096, + "inactive_split_bytes_peak": 7373883392, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:17.190169Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5925118976, + "inactive_split_bytes_current_delta": -387973120, + "inactive_split_bytes_peak": 7373883392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T134718068981Z.pickle", + "timestamp_utc": "2026-08-01T13:47:18.172225Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T134718068981Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T134718068981Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b322f5b756cdfe31161e88886fe61210dec4965 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T134718068981Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d0d3b6353bfc3a670b9b9335b541270e95c0c264989660f7a77feec54e093f6 +size 5566059 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T134814745530Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T134814745530Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0281b48174cd3b4c95c5183dcff2d42cb1a0306f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T134814745530Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803801088, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803801088, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66803801088, + "allocated_bytes_peak": 105818292736, + "device": 3, + "inactive_split_bytes_current": 3603883008, + "inactive_split_bytes_peak": 7373883392, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:48:14.745061Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803801088, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 3, + "inactive_split_bytes_current": 5925118976, + "inactive_split_bytes_peak": 7373883392, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:18.186895Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 3603883008, + "inactive_split_bytes_current_delta": -2321235968, + "inactive_split_bytes_peak": 7373883392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T134814745530Z.pickle", + "timestamp_utc": "2026-08-01T13:48:14.952261Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T134814745530Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T134814745530Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e8d1f3e8b3c6bbf6583da2f9bac03c8b24fc59ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T134814745530Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2112a344c6584be58b26283fc54f6a42d48d6940e23b7195eaccfc3916ad3355 +size 9207177 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T134709572236Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T134709572236Z.json new file mode 100644 index 0000000000000000000000000000000000000000..615d404ef143b88723d7e7d07dc92c806e113285 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T134709572236Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854868992, + "active_bytes_current_delta": 31622971392, + "active_bytes_peak": 82178515968, + "active_bytes_peak_delta": 46711737344, + "after": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 82178515968, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 82178515968, + "device": 4, + "inactive_split_bytes_current": 4897089536, + "inactive_split_bytes_peak": 7120315392, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 97475624960, + "reserved_bytes_peak": 97475624960, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T13:47:09.571832Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854868992, + "allocated_bytes_current_delta": 31622971392, + "allocated_bytes_peak": 82178515968, + "allocated_bytes_peak_delta": 46711737344, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:46:01.509032Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4897089536, + "inactive_split_bytes_current_delta": 4764712960, + "inactive_split_bytes_peak": 7120315392, + "inactive_split_bytes_peak_delta": 6966905856, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 97475624960, + "reserved_bytes_current_delta": 58231619584, + "reserved_bytes_peak": 97475624960, + "reserved_bytes_peak_delta": 58231619584, + "segment_count_current": 488, + "segment_count_current_delta": 417, + "segment_count_peak": 488, + "segment_count_peak_delta": 417, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T134709572236Z.pickle", + "timestamp_utc": "2026-08-01T13:47:09.681287Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T134709572236Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T134709572236Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..34c08c55d1580239ca5769fed930683ea11c71a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T134709572236Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cff91ce5186535b0e04ebb905bd0c1ee9299403adbecd092b041035338753555 +size 5377547 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T134710616399Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T134710616399Z.json new file mode 100644 index 0000000000000000000000000000000000000000..85ddd51d46730cb0573a561e1ca3c525793d159a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T134710616399Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854868992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105812307968, + "active_bytes_peak_delta": 23633792000, + "after": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105812307968, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105812307968, + "device": 4, + "inactive_split_bytes_current": 5077444608, + "inactive_split_bytes_peak": 7120315392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:10.615988Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854868992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105812307968, + "allocated_bytes_peak_delta": 23633792000, + "before": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 82178515968, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 82178515968, + "device": 4, + "inactive_split_bytes_current": 4897089536, + "inactive_split_bytes_peak": 7120315392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 97475624960, + "reserved_bytes_peak": 97475624960, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T13:47:09.698799Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5077444608, + "inactive_split_bytes_current_delta": 180355072, + "inactive_split_bytes_peak": 7120315392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107005083648, + "reserved_bytes_current_delta": 9529458688, + "reserved_bytes_peak": 107005083648, + "reserved_bytes_peak_delta": 9529458688, + "segment_count_current": 545, + "segment_count_current_delta": 57, + "segment_count_peak": 545, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T134710616399Z.pickle", + "timestamp_utc": "2026-08-01T13:47:10.725752Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T134710616399Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T134710616399Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c86410307eaed12a945ed779dcff8a440a429be2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T134710616399Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e657be54b5c1e402b4bf84f8c6453e2101e22eb60aa52d0e96fb37e43f04068f +size 5572197 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T134711600123Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T134711600123Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80be6c2f379f7670b81079da8bf4b82cfeaacf26 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T134711600123Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854868992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105812439040, + "active_bytes_peak_delta": 131072, + "after": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105812439040, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105812439040, + "device": 4, + "inactive_split_bytes_current": 5004044288, + "inactive_split_bytes_peak": 7120315392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:11.599724Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854868992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105812439040, + "allocated_bytes_peak_delta": 131072, + "before": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105812307968, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105812307968, + "device": 4, + "inactive_split_bytes_current": 5077444608, + "inactive_split_bytes_peak": 7120315392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:10.739189Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5004044288, + "inactive_split_bytes_current_delta": -73400320, + "inactive_split_bytes_peak": 7120315392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107005083648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107005083648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T134711600123Z.pickle", + "timestamp_utc": "2026-08-01T13:47:11.701222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T134711600123Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T134711600123Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4f9cb3eef69b5cccb3038faec1966ec04f9b72ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T134711600123Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0462e21e77bc74d37e6390b086e98c61076c54daec23de25a6b89510b7f48641 +size 5577400 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T134712614638Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T134712614638Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ba655246676f4591ea3260023c35d75c8eedfef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T134712614638Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854868992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813291520, + "active_bytes_peak_delta": 852480, + "after": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 5104707584, + "inactive_split_bytes_peak": 7248826368, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:12.614227Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854868992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813291520, + "allocated_bytes_peak_delta": 852480, + "before": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105812439040, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105812439040, + "device": 4, + "inactive_split_bytes_current": 5004044288, + "inactive_split_bytes_peak": 7120315392, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:11.713984Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5104707584, + "inactive_split_bytes_current_delta": 100663296, + "inactive_split_bytes_peak": 7248826368, + "inactive_split_bytes_peak_delta": 128510976, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107005083648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107005083648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T134712614638Z.pickle", + "timestamp_utc": "2026-08-01T13:47:12.712616Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T134712614638Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T134712614638Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27ee0bcef6a12a07c780f8297221c95a9f2704ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T134712614638Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16cb242503773845d2cfa135a0957b340b775ba5e51ac73439025cbe4e2df71e +size 5573815 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T134714068725Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T134714068725Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e26902bdaa6b5236e726a6446cf4153d4cb4fc1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T134714068725Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854868992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813291520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 5280868352, + "inactive_split_bytes_peak": 7248826368, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:14.068322Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854868992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813291520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 5104707584, + "inactive_split_bytes_peak": 7248826368, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:12.725281Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5280868352, + "inactive_split_bytes_current_delta": 176160768, + "inactive_split_bytes_peak": 7248826368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107005083648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107005083648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T134714068725Z.pickle", + "timestamp_utc": "2026-08-01T13:47:14.172021Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T134714068725Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T134714068725Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4af5f63406fd648247ab1a8371a8ec15405fb760 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T134714068725Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a67983bf972fce3637941c059bf6eef7052a633c9755fc110a69fc9ae6cbb873 +size 5597488 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T134715069771Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T134715069771Z.json new file mode 100644 index 0000000000000000000000000000000000000000..356b1c723836a2f3e2ab7dad6239796d3b6dbabf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T134715069771Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854868992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813291520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 5129873408, + "inactive_split_bytes_peak": 7248826368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:15.069366Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854868992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813291520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 5280868352, + "inactive_split_bytes_peak": 7248826368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:14.184945Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5129873408, + "inactive_split_bytes_current_delta": -150994944, + "inactive_split_bytes_peak": 7248826368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107005083648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107005083648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T134715069771Z.pickle", + "timestamp_utc": "2026-08-01T13:47:15.168060Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T134715069771Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T134715069771Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..81ce65dac84ab8390e50ba654bab6d0a4c442e29 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T134715069771Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d2ad8d33e839bfdf1d7b2f0c0d67371a5a4032778b5a0a0b234056262d73bc4 +size 5606120 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T134716067302Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T134716067302Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b6d0f5cc2f9b2ba9952bb6a787900c52bb324b6e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T134716067302Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854868992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813291520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 5429766144, + "inactive_split_bytes_peak": 7248826368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:16.066903Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854868992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813291520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 5129873408, + "inactive_split_bytes_peak": 7248826368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:15.180636Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5429766144, + "inactive_split_bytes_current_delta": 299892736, + "inactive_split_bytes_peak": 7248826368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107005083648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107005083648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T134716067302Z.pickle", + "timestamp_utc": "2026-08-01T13:47:16.169139Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T134716067302Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T134716067302Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e7ac711f8e8b5b0d4d19d79e786f5e376d5484c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T134716067302Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6677bfbd81e4a405a9e8511b7f4b0cf90c77c1c32fb57ba40bbdbad2a953675 +size 5606098 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T134717073838Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T134717073838Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4ebc0cc21e3ff25644ddd96eb759e98e7272c724 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T134717073838Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854868992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813291520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 5750630400, + "inactive_split_bytes_peak": 7248826368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:17.073428Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854868992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813291520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 5429766144, + "inactive_split_bytes_peak": 7248826368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:16.185301Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5750630400, + "inactive_split_bytes_current_delta": 320864256, + "inactive_split_bytes_peak": 7248826368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107005083648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107005083648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T134717073838Z.pickle", + "timestamp_utc": "2026-08-01T13:47:17.173561Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T134717073838Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T134717073838Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0425b730e981b091c6e7007add9d9a8e969c9b96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T134717073838Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:709ec807fa79e75ba2a0e7d01d13338e8ef598c5227b69406f558103ad5782e3 +size 5601536 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T134718068965Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T134718068965Z.json new file mode 100644 index 0000000000000000000000000000000000000000..802b16e2267dba3c60a1e48bd7aed1a244b13301 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T134718068965Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854868992, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813291520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 5016627200, + "inactive_split_bytes_peak": 7248826368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:18.068574Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854868992, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813291520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 5750630400, + "inactive_split_bytes_peak": 7248826368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:17.186179Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5016627200, + "inactive_split_bytes_current_delta": -734003200, + "inactive_split_bytes_peak": 7248826368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107005083648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107005083648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T134718068965Z.pickle", + "timestamp_utc": "2026-08-01T13:47:18.166488Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T134718068965Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T134718068965Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d5ae597b88ec178d15428c6d6dee44fffff2a6dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T134718068965Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b59c5ecd2e106d3aa53a79eddd6a6e05ec9b694770eb674753d4d45a8503da1 +size 5572535 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T134814745620Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T134814745620Z.json new file mode 100644 index 0000000000000000000000000000000000000000..635aa88c1124530e1b72cd75bea6e1b68e183c3d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T134814745620Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804226048, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105813291520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66804226048, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66804226048, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 3580389376, + "inactive_split_bytes_peak": 7248826368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:48:14.745058Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804226048, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105813291520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854868992, + "active_bytes_peak": 105813291520, + "allocated_bytes_current": 66854868992, + "allocated_bytes_peak": 105813291520, + "device": 4, + "inactive_split_bytes_current": 5016627200, + "inactive_split_bytes_peak": 7248826368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107005083648, + "reserved_bytes_peak": 107005083648, + "segment_count_current": 545, + "segment_count_peak": 545, + "timestamp_utc": "2026-08-01T13:47:18.179075Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 3580389376, + "inactive_split_bytes_current_delta": -1436237824, + "inactive_split_bytes_peak": 7248826368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107005083648, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107005083648, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 545, + "segment_count_current_delta": 0, + "segment_count_peak": 545, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T134814745620Z.pickle", + "timestamp_utc": "2026-08-01T13:48:14.948643Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T134814745620Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T134814745620Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e0adf092fc00e0307d27c2f5f5f7702262cfcd0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T134814745620Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:960f5742f1b7afa00cd4ccbb2b301bb16c0d1f0dbbbed0631144b268954a8bbf +size 9336887 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T134709572271Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T134709572271Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2218b950ec2c85f7b634cfd5664c57d37458611f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T134709572271Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 31622988800, + "active_bytes_peak": 82174059520, + "active_bytes_peak_delta": 46707723264, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 82174059520, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 82174059520, + "device": 5, + "inactive_split_bytes_current": 5275001856, + "inactive_split_bytes_peak": 7079567360, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:47:09.571681Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 31622988800, + "allocated_bytes_peak": 82174059520, + "allocated_bytes_peak_delta": 46707723264, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:46:01.509382Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5275001856, + "inactive_split_bytes_current_delta": 5142182912, + "inactive_split_bytes_peak": 7079567360, + "inactive_split_bytes_peak_delta": 6919181312, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 491, + "segment_count_current_delta": 420, + "segment_count_peak": 491, + "segment_count_peak_delta": 420, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T134709572271Z.pickle", + "timestamp_utc": "2026-08-01T13:47:09.684689Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T134709572271Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T134709572271Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6997b895e15c358a8d5e9ed577a5f62b8ae1153f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T134709572271Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42b7e94040cd562bd4045b59ddf653940ff14204676b95f24a8262fcbb6c2b63 +size 5369992 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T134710616353Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T134710616353Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94edefe565cb29e85ec3b343f174cd35347a65ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T134710616353Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813069824, + "active_bytes_peak_delta": 23639010304, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813069824, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813069824, + "device": 5, + "inactive_split_bytes_current": 5344207872, + "inactive_split_bytes_peak": 7079567360, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:10.615940Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813069824, + "allocated_bytes_peak_delta": 23639010304, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 82174059520, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 82174059520, + "device": 5, + "inactive_split_bytes_current": 5275001856, + "inactive_split_bytes_peak": 7079567360, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:47:09.704213Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5344207872, + "inactive_split_bytes_current_delta": 69206016, + "inactive_split_bytes_peak": 7079567360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 9521070080, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 9521070080, + "segment_count_current": 546, + "segment_count_current_delta": 55, + "segment_count_peak": 546, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T134710616353Z.pickle", + "timestamp_utc": "2026-08-01T13:47:10.728133Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T134710616353Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T134710616353Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1cd109034a3c7ea3a386a3b6d038eb40df5547b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T134710616353Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:299aeee251b134b0f3a3cf88d34566eaff02580b605208d6c049d55b76332dab +size 5600563 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T134711600146Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T134711600146Z.json new file mode 100644 index 0000000000000000000000000000000000000000..072fe7acd35c055a30387ad4f4d9b64eedb06c49 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T134711600146Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813069824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813069824, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813069824, + "device": 5, + "inactive_split_bytes_current": 5224670208, + "inactive_split_bytes_peak": 7079567360, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:11.599744Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813069824, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813069824, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813069824, + "device": 5, + "inactive_split_bytes_current": 5344207872, + "inactive_split_bytes_peak": 7079567360, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:10.743338Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5224670208, + "inactive_split_bytes_current_delta": -119537664, + "inactive_split_bytes_peak": 7079567360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T134711600146Z.pickle", + "timestamp_utc": "2026-08-01T13:47:11.705081Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T134711600146Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T134711600146Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..65c323c4a734a502035869d304630dee8d1be876 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T134711600146Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2acb310438f86324896a7ff2db0f4add30d2b54c7b8bc38ebc0482e2dc79994e +size 5586703 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T134712614625Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T134712614625Z.json new file mode 100644 index 0000000000000000000000000000000000000000..143acc964909c8dd8934e626ec9a06622b60481e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T134712614625Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813966848, + "active_bytes_peak_delta": 897024, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813966848, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813966848, + "device": 5, + "inactive_split_bytes_current": 6105474048, + "inactive_split_bytes_peak": 7160159232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:12.614217Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813966848, + "allocated_bytes_peak_delta": 897024, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813069824, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813069824, + "device": 5, + "inactive_split_bytes_current": 5224670208, + "inactive_split_bytes_peak": 7079567360, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:11.719855Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6105474048, + "inactive_split_bytes_current_delta": 880803840, + "inactive_split_bytes_peak": 7160159232, + "inactive_split_bytes_peak_delta": 80591872, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T134712614625Z.pickle", + "timestamp_utc": "2026-08-01T13:47:12.722194Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T134712614625Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T134712614625Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..54efeccf6a215c4b747dae8a051052225060e81a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T134712614625Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef55eeddaf4a5eecb6d95d267142d106ecaf77a1944ff7a5f948ff4afcabd659 +size 5567433 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T134714068798Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T134714068798Z.json new file mode 100644 index 0000000000000000000000000000000000000000..98340b502700b1e1df3d9a49cdd6460085262d56 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T134714068798Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815712256, + "active_bytes_peak_delta": 1745408, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105815712256, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105815712256, + "device": 5, + "inactive_split_bytes_current": 5929313280, + "inactive_split_bytes_peak": 7238933504, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:14.068302Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815712256, + "allocated_bytes_peak_delta": 1745408, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813966848, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813966848, + "device": 5, + "inactive_split_bytes_current": 6105474048, + "inactive_split_bytes_peak": 7160159232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:12.739512Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5929313280, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 7238933504, + "inactive_split_bytes_peak_delta": 78774272, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T134714068798Z.pickle", + "timestamp_utc": "2026-08-01T13:47:14.174688Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T134714068798Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T134714068798Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ca26fcf46d47e3cfdfcda4f3f21da2260a50125 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T134714068798Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94268d750759ce2a505c1546696881d807ea260f1d5a72d53229d19d03f08f24 +size 5566597 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T134715069731Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T134715069731Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ce14665029c849d5830891ba018676c42918fb8f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T134715069731Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815712256, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105815712256, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105815712256, + "device": 5, + "inactive_split_bytes_current": 5373568000, + "inactive_split_bytes_peak": 7238933504, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:15.069332Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815712256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105815712256, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105815712256, + "device": 5, + "inactive_split_bytes_current": 5929313280, + "inactive_split_bytes_peak": 7238933504, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:14.189305Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5373568000, + "inactive_split_bytes_current_delta": -555745280, + "inactive_split_bytes_peak": 7238933504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T134715069731Z.pickle", + "timestamp_utc": "2026-08-01T13:47:15.171576Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T134715069731Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T134715069731Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f092d3b9b0a04a30d9e5aaaee7c0967a014ef486 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T134715069731Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3fce33ec06e3ddb95db41f609f6bc29b18919f959a24b92eff746f3fe6b1dd0 +size 5567222 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T134716067302Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T134716067302Z.json new file mode 100644 index 0000000000000000000000000000000000000000..68814374ddc2748b965386b019ff9f6eb53f68f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T134716067302Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 2580480, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 5, + "inactive_split_bytes_current": 5283390464, + "inactive_split_bytes_peak": 7238933504, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:16.066891Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 2580480, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105815712256, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105815712256, + "device": 5, + "inactive_split_bytes_current": 5373568000, + "inactive_split_bytes_peak": 7238933504, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:15.185926Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5283390464, + "inactive_split_bytes_current_delta": -90177536, + "inactive_split_bytes_peak": 7238933504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T134716067302Z.pickle", + "timestamp_utc": "2026-08-01T13:47:16.171322Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T134716067302Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T134716067302Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1554b6bc94a5508c360d86d5539e611f08f70649 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T134716067302Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:614abaa192fb7ff1bdbe75627a4b9811c618f0899912dcc5a992b0a95f4f76c8 +size 5575808 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T134717073850Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T134717073850Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27ebd6c1edb36c1ebbe7da1ad43983f0cc059ffa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T134717073850Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 5, + "inactive_split_bytes_current": 5751055360, + "inactive_split_bytes_peak": 7238933504, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:17.073410Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 5, + "inactive_split_bytes_current": 5283390464, + "inactive_split_bytes_peak": 7238933504, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:16.186222Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5751055360, + "inactive_split_bytes_current_delta": 467664896, + "inactive_split_bytes_peak": 7238933504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T134717073850Z.pickle", + "timestamp_utc": "2026-08-01T13:47:17.175002Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T134717073850Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T134717073850Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..15f841a2c400adc4b6045952c17e842b9d0c2fb9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T134717073850Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c89dda1ec9da5489b516c1422d1f6a35a5ced3b535a67d7a7c885a85f612746 +size 5567136 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T134718068962Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T134718068962Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2eba798f953c8b35ff4f03e16df4786318fedec1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T134718068962Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 5, + "inactive_split_bytes_current": 5459551232, + "inactive_split_bytes_peak": 7238933504, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:18.068550Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 5, + "inactive_split_bytes_current": 5751055360, + "inactive_split_bytes_peak": 7238933504, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:17.189380Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5459551232, + "inactive_split_bytes_current_delta": -291504128, + "inactive_split_bytes_peak": 7238933504, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T134718068962Z.pickle", + "timestamp_utc": "2026-08-01T13:47:18.169217Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T134718068962Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T134718068962Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..618eb4e55e366f9524fd49e104af71ec1ba7c59a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T134718068962Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:374fa1180e09938091b35ad87eabaea7a92c2afc83959c9b111986dcb29e2f51 +size 5565687 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T134814745541Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T134814745541Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e8a07572b0f304d64dc6c489c8c5ea411c6c507 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T134814745541Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803801088, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803801088, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66803801088, + "allocated_bytes_peak": 105818292736, + "device": 5, + "inactive_split_bytes_current": 3601785856, + "inactive_split_bytes_peak": 7265196032, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:48:14.745061Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803801088, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 5, + "inactive_split_bytes_current": 5459551232, + "inactive_split_bytes_peak": 7238933504, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:18.183769Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 3601785856, + "inactive_split_bytes_current_delta": -1857765376, + "inactive_split_bytes_peak": 7265196032, + "inactive_split_bytes_peak_delta": 26262528, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T134814745541Z.pickle", + "timestamp_utc": "2026-08-01T13:48:14.950693Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T134814745541Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T134814745541Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d078ced3653cd16b3c4dde76857d0a37f539bfb6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T134814745541Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dd3dfaf01617709eafbb54083a24423ba9c78cec0297666ea35cb90d6d1e4ed +size 9207514 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T134709572370Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T134709572370Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5c19886339d50edc361ae191b128f3baf8c397a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T134709572370Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 31622988800, + "active_bytes_peak": 82174059520, + "active_bytes_peak_delta": 46707723264, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 82174059520, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 82174059520, + "device": 6, + "inactive_split_bytes_current": 5277099008, + "inactive_split_bytes_peak": 7081664512, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:47:09.571791Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 31622988800, + "allocated_bytes_peak": 82174059520, + "allocated_bytes_peak_delta": 46707723264, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:46:01.509384Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5277099008, + "inactive_split_bytes_current_delta": 5144280064, + "inactive_split_bytes_peak": 7081664512, + "inactive_split_bytes_peak_delta": 6921278464, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 491, + "segment_count_current_delta": 420, + "segment_count_peak": 491, + "segment_count_peak_delta": 420, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T134709572370Z.pickle", + "timestamp_utc": "2026-08-01T13:47:09.687266Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T134709572370Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T134709572370Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2211fb2f1a25130efca9d0edd0257ecbe6eb68df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T134709572370Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be4193a39643cf0531384068dfd291ad66ba5ba887eda843cda14e5cc3059fe2 +size 5370091 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T134710616350Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T134710616350Z.json new file mode 100644 index 0000000000000000000000000000000000000000..795d8f9a8b25d9fa3c2502fe9f73a1f5c9c4cafb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T134710616350Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813069824, + "active_bytes_peak_delta": 23639010304, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813069824, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813069824, + "device": 6, + "inactive_split_bytes_current": 5453259776, + "inactive_split_bytes_peak": 7081664512, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:10.615922Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813069824, + "allocated_bytes_peak_delta": 23639010304, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 82174059520, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 82174059520, + "device": 6, + "inactive_split_bytes_current": 5277099008, + "inactive_split_bytes_peak": 7081664512, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:47:09.706458Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5453259776, + "inactive_split_bytes_current_delta": 176160768, + "inactive_split_bytes_peak": 7081664512, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 9521070080, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 9521070080, + "segment_count_current": 546, + "segment_count_current_delta": 55, + "segment_count_peak": 546, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T134710616350Z.pickle", + "timestamp_utc": "2026-08-01T13:47:10.728350Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T134710616350Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T134710616350Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..910938008861365425b54013a20399306845d81c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T134710616350Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33576f577dccc68e054b1f3e81a6d2ab3866abc6f7d10301db10a443676b9cf2 +size 5561433 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T134711600083Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T134711600083Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e431f3a041c81022fc7e5d85695e085dd79bf0ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T134711600083Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814119424, + "active_bytes_peak_delta": 1049600, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105814119424, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105814119424, + "device": 6, + "inactive_split_bytes_current": 5633614848, + "inactive_split_bytes_peak": 7347895296, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:11.599693Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814119424, + "allocated_bytes_peak_delta": 1049600, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105813069824, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105813069824, + "device": 6, + "inactive_split_bytes_current": 5453259776, + "inactive_split_bytes_peak": 7081664512, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:10.743539Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5633614848, + "inactive_split_bytes_current_delta": 180355072, + "inactive_split_bytes_peak": 7347895296, + "inactive_split_bytes_peak_delta": 266230784, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T134711600083Z.pickle", + "timestamp_utc": "2026-08-01T13:47:11.704701Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T134711600083Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T134711600083Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3aa88cea584087389435010ce0935addb8745cdc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T134711600083Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:805f01c1fdb69125f6b347f4855596b6762573d1e4348743dd6190f464f03688 +size 5567408 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T134712614580Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T134712614580Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef0704d0e77ddfcbb6d0c7a7d992b2996f73971c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T134712614580Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105816257024, + "active_bytes_peak_delta": 2137600, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 6, + "inactive_split_bytes_current": 6227108864, + "inactive_split_bytes_peak": 7347895296, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:12.614196Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105816257024, + "allocated_bytes_peak_delta": 2137600, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105814119424, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105814119424, + "device": 6, + "inactive_split_bytes_current": 5633614848, + "inactive_split_bytes_peak": 7347895296, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:11.719374Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6227108864, + "inactive_split_bytes_current_delta": 593494016, + "inactive_split_bytes_peak": 7347895296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T134712614580Z.pickle", + "timestamp_utc": "2026-08-01T13:47:12.717074Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T134712614580Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T134712614580Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2445e78eaa08b27ce538d9c1ae9347651e3419d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T134712614580Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d155f5f29ab3049fe53968b6430e6ba01a2c4ad6899592fc5133d34629b3895 +size 5566402 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T134714068699Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T134714068699Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f1d8500b56011a90d62147bf6d6fef98c7079c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T134714068699Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105816257024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 6, + "inactive_split_bytes_current": 5927216128, + "inactive_split_bytes_peak": 7347895296, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:14.068281Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105816257024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 6, + "inactive_split_bytes_current": 6227108864, + "inactive_split_bytes_peak": 7347895296, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:12.731557Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5927216128, + "inactive_split_bytes_current_delta": -299892736, + "inactive_split_bytes_peak": 7347895296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T134714068699Z.pickle", + "timestamp_utc": "2026-08-01T13:47:14.173335Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T134714068699Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T134714068699Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb94428016ae1cccd16f43144682cb21c02299ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T134714068699Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15241d8a07e863802b77dd85d5a2806495cc5471fbe882fe864bcb6c7ae1a3ab +size 5584696 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T134715069751Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T134715069751Z.json new file mode 100644 index 0000000000000000000000000000000000000000..86615458145919e9e23f9fb27dd08aa785d84c63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T134715069751Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105816257024, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 6, + "inactive_split_bytes_current": 5373568000, + "inactive_split_bytes_peak": 7347895296, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:15.069333Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105816257024, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 6, + "inactive_split_bytes_current": 5927216128, + "inactive_split_bytes_peak": 7347895296, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:14.187791Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5373568000, + "inactive_split_bytes_current_delta": -553648128, + "inactive_split_bytes_peak": 7347895296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T134715069751Z.pickle", + "timestamp_utc": "2026-08-01T13:47:15.171448Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T134715069751Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T134715069751Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a9d5a7befd46a1d7131c5545d8457f5bf752a65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T134715069751Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4003a4ae9dedd326566eda071c17c7b53c43d815417a294a8d6408369c862ee6 +size 5567333 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T134716067344Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T134716067344Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06b690dbe96eaf2e0141db1b50ca2653c324e5ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T134716067344Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 2035712, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 6, + "inactive_split_bytes_current": 6313092096, + "inactive_split_bytes_peak": 7347895296, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:16.066927Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 2035712, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105816257024, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105816257024, + "device": 6, + "inactive_split_bytes_current": 5373568000, + "inactive_split_bytes_peak": 7347895296, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:15.185782Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6313092096, + "inactive_split_bytes_current_delta": 939524096, + "inactive_split_bytes_peak": 7347895296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T134716067344Z.pickle", + "timestamp_utc": "2026-08-01T13:47:16.170254Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T134716067344Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T134716067344Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fc74b88f7026411150fd32f4dd34049f03fbc015 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T134716067344Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eae81bfffbf75b12c3f5fd1345d06477e887d6c4e9d83ae868193a55294d5fc +size 5562730 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T134717073785Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T134717073785Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e5446237e3f5adb38504a08ca1a93c2d872faef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T134717073785Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 6, + "inactive_split_bytes_current": 5929313280, + "inactive_split_bytes_peak": 7347895296, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:17.073388Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 6, + "inactive_split_bytes_current": 6313092096, + "inactive_split_bytes_peak": 7347895296, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:16.185073Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5929313280, + "inactive_split_bytes_current_delta": -383778816, + "inactive_split_bytes_peak": 7347895296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T134717073785Z.pickle", + "timestamp_utc": "2026-08-01T13:47:17.176066Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T134717073785Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T134717073785Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..96863bef0a3ad31e130c25f781dd8a84c966c779 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T134717073785Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12dcdf2ed49c9be0d844f67d161f0286eb81e6d8ce8582467599edf25f6922ba +size 5565711 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T134718068941Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T134718068941Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3663bc7793901ec1a7590ab2165d515c61deefc5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T134718068941Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444032, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 6, + "inactive_split_bytes_current": 5369373696, + "inactive_split_bytes_peak": 7347895296, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:18.068521Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444032, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 6, + "inactive_split_bytes_current": 5929313280, + "inactive_split_bytes_peak": 7347895296, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:17.190406Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5369373696, + "inactive_split_bytes_current_delta": -559939584, + "inactive_split_bytes_peak": 7347895296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T134718068941Z.pickle", + "timestamp_utc": "2026-08-01T13:47:18.169015Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T134718068941Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T134718068941Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b06c17c1f04dde90bb4504c82f5235fd56035772 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T134718068941Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b067174d7ad99133b61a5d52692d138a9fd5e218af85b11cedf28373372732a9 +size 5567186 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T134814745515Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T134814745515Z.json new file mode 100644 index 0000000000000000000000000000000000000000..991d722e5eed6f5dec761186f6b7b1be0405d333 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T134814745515Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803801088, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105818292736, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803801088, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66803801088, + "allocated_bytes_peak": 105818292736, + "device": 6, + "inactive_split_bytes_current": 3601785856, + "inactive_split_bytes_peak": 7347895296, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:48:14.745060Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803801088, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105818292736, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444032, + "active_bytes_peak": 105818292736, + "allocated_bytes_current": 66854444032, + "allocated_bytes_peak": 105818292736, + "device": 6, + "inactive_split_bytes_current": 5369373696, + "inactive_split_bytes_peak": 7347895296, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:18.183535Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 3601785856, + "inactive_split_bytes_current_delta": -1767587840, + "inactive_split_bytes_peak": 7347895296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T134814745515Z.pickle", + "timestamp_utc": "2026-08-01T13:48:14.943636Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T134814745515Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T134814745515Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..40106c74cd94d96f52cc79d9dbe1cccfed3eae09 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T134814745515Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06c5a672fa18abfaab0172a507b5a4bc0c0ff24d01e5972e5ae70cc8d73f60b7 +size 9315008 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T134709572447Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T134709572447Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d8102baebe34e3ec28e904aa05d390c3ef69d861 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T134709572447Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444544, + "active_bytes_current_delta": 31622989312, + "active_bytes_peak": 82174059520, + "active_bytes_peak_delta": 46707723264, + "after": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 82174059520, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 82174059520, + "device": 7, + "inactive_split_bytes_current": 5279195648, + "inactive_split_bytes_peak": 7083761664, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:47:09.571968Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444544, + "allocated_bytes_current_delta": 31622989312, + "allocated_bytes_peak": 82174059520, + "allocated_bytes_peak_delta": 46707723264, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:46:01.509961Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5279195648, + "inactive_split_bytes_current_delta": 5146376704, + "inactive_split_bytes_peak": 7083761664, + "inactive_split_bytes_peak_delta": 6923375616, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 97488207872, + "reserved_bytes_current_delta": 58244202496, + "reserved_bytes_peak": 97488207872, + "reserved_bytes_peak_delta": 58244202496, + "segment_count_current": 491, + "segment_count_current_delta": 420, + "segment_count_peak": 491, + "segment_count_peak_delta": 420, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T134709572447Z.pickle", + "timestamp_utc": "2026-08-01T13:47:09.684122Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T134709572447Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T134709572447Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea653468f2bbfc34a7226d3acae91c2daeb814ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T134709572447Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb8d9ed18b37abae6b6bc2a4a1669948a7637ffea76b2bedf9648dd6276dd57c +size 5374025 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T134710616476Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T134710616476Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3898e4b14876ca5862b497bf77a3b93db825ebf5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T134710616476Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813070336, + "active_bytes_peak_delta": 23639010816, + "after": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105813070336, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105813070336, + "device": 7, + "inactive_split_bytes_current": 5518270976, + "inactive_split_bytes_peak": 7083761664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:10.616062Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813070336, + "allocated_bytes_peak_delta": 23639010816, + "before": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 82174059520, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 82174059520, + "device": 7, + "inactive_split_bytes_current": 5279195648, + "inactive_split_bytes_peak": 7083761664, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 97488207872, + "reserved_bytes_peak": 97488207872, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:47:09.701546Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5518270976, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 7083761664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 9521070080, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 9521070080, + "segment_count_current": 546, + "segment_count_current_delta": 55, + "segment_count_peak": 546, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T134710616476Z.pickle", + "timestamp_utc": "2026-08-01T13:47:10.725487Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T134710616476Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T134710616476Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e024b60d8c0da8e869c65fac03df53561ed215f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T134710616476Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfcb676f1c8c70564201a93347210ecbb559690b19ebd2ace722e2901b653eb0 +size 5560310 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T134711600223Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T134711600223Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d987d76ae50dcbb44ca69a523d2b3b8a3baea3fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T134711600223Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814443008, + "active_bytes_peak_delta": 1372672, + "after": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105814443008, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105814443008, + "device": 7, + "inactive_split_bytes_current": 5195309568, + "inactive_split_bytes_peak": 7083761664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:11.599814Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814443008, + "allocated_bytes_peak_delta": 1372672, + "before": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105813070336, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105813070336, + "device": 7, + "inactive_split_bytes_current": 5518270976, + "inactive_split_bytes_peak": 7083761664, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:10.740610Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5195309568, + "inactive_split_bytes_current_delta": -322961408, + "inactive_split_bytes_peak": 7083761664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T134711600223Z.pickle", + "timestamp_utc": "2026-08-01T13:47:11.702503Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T134711600223Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T134711600223Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f0fb768ce6ce26e7d7d9c9b36f69aa1ec9490f47 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T134711600223Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:509a79dd223c4ecca618658a88ca5157da79cc6d66e26c55e5bd49248b08d487 +size 5569571 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T134712614752Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T134712614752Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2745762305f5a20532321ac0c70cc2959b3ab9e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T134712614752Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814443008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105814443008, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105814443008, + "device": 7, + "inactive_split_bytes_current": 6105473536, + "inactive_split_bytes_peak": 7160158720, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:12.614332Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814443008, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105814443008, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105814443008, + "device": 7, + "inactive_split_bytes_current": 5195309568, + "inactive_split_bytes_peak": 7083761664, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:11.715084Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6105473536, + "inactive_split_bytes_current_delta": 910163968, + "inactive_split_bytes_peak": 7160158720, + "inactive_split_bytes_peak_delta": 76397056, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T134712614752Z.pickle", + "timestamp_utc": "2026-08-01T13:47:12.713740Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T134712614752Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T134712614752Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..63e44077107b8bbd55c8397f09d71f29ffcf55cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T134712614752Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c09060536d691e004fcbf5c08fc11bec85b1713ddb8bfde97983934dc767310 +size 5569205 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T134714068847Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T134714068847Z.json new file mode 100644 index 0000000000000000000000000000000000000000..53b29be9c43c83144a9ef1207f20ce6a18dbaa85 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T134714068847Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815712768, + "active_bytes_peak_delta": 1269760, + "after": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105815712768, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105815712768, + "device": 7, + "inactive_split_bytes_current": 5929312768, + "inactive_split_bytes_peak": 7236835840, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:14.068418Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815712768, + "allocated_bytes_peak_delta": 1269760, + "before": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105814443008, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105814443008, + "device": 7, + "inactive_split_bytes_current": 6105473536, + "inactive_split_bytes_peak": 7160158720, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:12.726125Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5929312768, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 7236835840, + "inactive_split_bytes_peak_delta": 76677120, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T134714068847Z.pickle", + "timestamp_utc": "2026-08-01T13:47:14.171057Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T134714068847Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T134714068847Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..355cdefce460a951f243c696d3c344261200cd8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T134714068847Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41f5cb189d714cdc28bca7cae5f52bb8d9b5a3abf6f2a7a6c917c864af728b04 +size 5585905 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T134715069849Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T134715069849Z.json new file mode 100644 index 0000000000000000000000000000000000000000..daf4d7bbc0f9226d96f325407c1cb1d66d2bf302 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T134715069849Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815712768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105815712768, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105815712768, + "device": 7, + "inactive_split_bytes_current": 5258224128, + "inactive_split_bytes_peak": 7236835840, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:15.069450Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815712768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105815712768, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105815712768, + "device": 7, + "inactive_split_bytes_current": 5929312768, + "inactive_split_bytes_peak": 7236835840, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:14.183969Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5258224128, + "inactive_split_bytes_current_delta": -671088640, + "inactive_split_bytes_peak": 7236835840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T134715069849Z.pickle", + "timestamp_utc": "2026-08-01T13:47:15.168706Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T134715069849Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T134715069849Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..458677c639d3029ded1c96d3e162f4e1c86dba0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T134715069849Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47c0738a5c765616f284b8ed46d9355b98b331eaa08a719e7b52594bf518463e +size 5569260 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T134716067430Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T134716067430Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c9fa8fa656ddb4632ca7f3dbd1452e2728b4b14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T134716067430Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818293248, + "active_bytes_peak_delta": 2580480, + "after": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105818293248, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105818293248, + "device": 7, + "inactive_split_bytes_current": 6310994432, + "inactive_split_bytes_peak": 7236835840, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:16.067017Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818293248, + "allocated_bytes_peak_delta": 2580480, + "before": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105815712768, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105815712768, + "device": 7, + "inactive_split_bytes_current": 5258224128, + "inactive_split_bytes_peak": 7236835840, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:15.181108Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6310994432, + "inactive_split_bytes_current_delta": 1052770304, + "inactive_split_bytes_peak": 7236835840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T134716067430Z.pickle", + "timestamp_utc": "2026-08-01T13:47:16.169260Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T134716067430Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T134716067430Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4da567817602bb234fae571939a5a58099d0bafe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T134716067430Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acbb7af2674be98815e83485b0093efda65f1b26af87f380342a815f94502c38 +size 5591662 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T134717073919Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T134717073919Z.json new file mode 100644 index 0000000000000000000000000000000000000000..39c21108ad4735d2af7a8ce13e2c1ac0e10c2a6e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T134717073919Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818293248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105818293248, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105818293248, + "device": 7, + "inactive_split_bytes_current": 5929312768, + "inactive_split_bytes_peak": 7236835840, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:17.073517Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818293248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105818293248, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105818293248, + "device": 7, + "inactive_split_bytes_current": 6310994432, + "inactive_split_bytes_peak": 7236835840, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:16.184886Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5929312768, + "inactive_split_bytes_current_delta": -381681664, + "inactive_split_bytes_peak": 7236835840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T134717073919Z.pickle", + "timestamp_utc": "2026-08-01T13:47:17.173262Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T134717073919Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T134717073919Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..75a6d3b860fbb150526b5b7da4588785f98155d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T134717073919Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:941ad5638a3114594264f6ee03cce230a815afedb30f618f7832975b423dfe44 +size 5567896 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T134718069123Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T134718069123Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc4d2e4e04069123e78d3c3acbbeda11d05cf534 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T134718069123Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854444544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105818293248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105818293248, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105818293248, + "device": 7, + "inactive_split_bytes_current": 5254029824, + "inactive_split_bytes_peak": 7236835840, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:18.068711Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854444544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105818293248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105818293248, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105818293248, + "device": 7, + "inactive_split_bytes_current": 5929312768, + "inactive_split_bytes_peak": 7236835840, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:17.185707Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5254029824, + "inactive_split_bytes_current_delta": -675282944, + "inactive_split_bytes_peak": 7236835840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T134718069123Z.pickle", + "timestamp_utc": "2026-08-01T13:47:18.167174Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T134718069123Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T134718069123Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e9480e1d647d42f694bbe72e516ee84f04719009 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T134718069123Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9833a6e2a398d45df41e7bafb67d1fde48328e856a7cfdda3723b7a65b087d3d +size 5569222 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T134814745509Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T134814745509Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f22289e613d734adfe0695dadbb0817e7871687c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T134814745509Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803801600, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105818293248, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803801600, + "active_bytes_peak": 105818293248, + "allocated_bytes_current": 66803801600, + "allocated_bytes_peak": 105818293248, + "device": 7, + "inactive_split_bytes_current": 3605979648, + "inactive_split_bytes_peak": 7236835840, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:48:14.745059Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803801600, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105818293248, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854444544, + "active_bytes_peak": 105818293248, + "allocated_bytes_current": 66854444544, + "allocated_bytes_peak": 105818293248, + "device": 7, + "inactive_split_bytes_current": 5254029824, + "inactive_split_bytes_peak": 7236835840, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107009277952, + "reserved_bytes_peak": 107009277952, + "segment_count_current": 546, + "segment_count_peak": 546, + "timestamp_utc": "2026-08-01T13:47:18.179497Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 3605979648, + "inactive_split_bytes_current_delta": -1648050176, + "inactive_split_bytes_peak": 7236835840, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107009277952, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107009277952, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 546, + "segment_count_current_delta": 0, + "segment_count_peak": 546, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T134814745509Z.pickle", + "timestamp_utc": "2026-08-01T13:48:14.940055Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T134814745509Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T134814745509Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..357da2bdf84590d8434e75ee420fb03a348e4dd2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T134814745509Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3767761358c695b65f0555f9dca6830d63e1ef6300d2e82c3602a30ea596b9f0 +size 9209354 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e5c76cb6f8a84e5b27832afe33693cf174dc5e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 2048 +capacity: 1.50 +expert_dispatch_overlap: off +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6033 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 226 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T135100155276Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T135100155276Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e7a6872d4b34e5a5690c0c022978254548889c63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T135100155276Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 82146977792, + "active_bytes_peak_delta": 46680641536, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82146977792, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82146977792, + "device": 0, + "inactive_split_bytes_current": 7360777216, + "inactive_split_bytes_peak": 7648708608, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 95323947008, + "reserved_bytes_peak": 95323947008, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:51:00.154695Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 82146977792, + "allocated_bytes_peak_delta": 46680641536, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:49:52.071664Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7360777216, + "inactive_split_bytes_current_delta": 7225861120, + "inactive_split_bytes_peak": 7648708608, + "inactive_split_bytes_peak_delta": 7488076800, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 95323947008, + "reserved_bytes_current_delta": 56077844480, + "reserved_bytes_peak": 95323947008, + "reserved_bytes_peak_delta": 56077844480, + "segment_count_current": 421, + "segment_count_current_delta": 349, + "segment_count_peak": 421, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T135100155276Z.pickle", + "timestamp_utc": "2026-08-01T13:51:00.272181Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T135100155276Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T135100155276Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a06bc86ba57018a6ecb95db507a0f9186a816925 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T135100155276Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa457d7347f39dcdd2806ad6aa247029c6411819a010642017e42ab0fd851a62 +size 5932821 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T135101378462Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T135101378462Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f8033d09cf07b1181d2ee3014c83cfbe3062da0f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T135101378462Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788781568, + "active_bytes_peak_delta": 23641803776, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 6542887936, + "inactive_split_bytes_peak": 8344271872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107097358336, + "reserved_bytes_peak": 107097358336, + "segment_count_current": 482, + "segment_count_peak": 482, + "timestamp_utc": "2026-08-01T13:51:01.378031Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788781568, + "allocated_bytes_peak_delta": 23641803776, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82146977792, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82146977792, + "device": 0, + "inactive_split_bytes_current": 7360777216, + "inactive_split_bytes_peak": 7648708608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 95323947008, + "reserved_bytes_peak": 95323947008, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T13:51:00.289399Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6542887936, + "inactive_split_bytes_current_delta": -817889280, + "inactive_split_bytes_peak": 8344271872, + "inactive_split_bytes_peak_delta": 695563264, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107097358336, + "reserved_bytes_current_delta": 11773411328, + "reserved_bytes_peak": 107097358336, + "reserved_bytes_peak_delta": 11773411328, + "segment_count_current": 482, + "segment_count_current_delta": 61, + "segment_count_peak": 482, + "segment_count_peak_delta": 61, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T135101378462Z.pickle", + "timestamp_utc": "2026-08-01T13:51:01.494853Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T135101378462Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T135101378462Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..380ced6d62a05a063ab626ee4a750a8daa0e138e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T135101378462Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c327f85827a7ebe1ffd19f2c45da4adc73978d3c9424e4223a05b566a147d7a +size 6074490 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T135102524536Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T135102524536Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3fced4b98696901f0a779a3413971f58ef54f20 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T135102524536Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788781568, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 6660328448, + "inactive_split_bytes_peak": 8757777408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:02.524140Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788781568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 6542887936, + "inactive_split_bytes_peak": 8344271872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107097358336, + "reserved_bytes_peak": 107097358336, + "segment_count_current": 482, + "segment_count_peak": 482, + "timestamp_utc": "2026-08-01T13:51:01.509157Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6660328448, + "inactive_split_bytes_current_delta": 117440512, + "inactive_split_bytes_peak": 8757777408, + "inactive_split_bytes_peak_delta": 413505536, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107567120384, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 483, + "segment_count_current_delta": 1, + "segment_count_peak": 483, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T135102524536Z.pickle", + "timestamp_utc": "2026-08-01T13:51:02.632498Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T135102524536Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T135102524536Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8fb505e983b3d9c6d5b1160c8ae3f9f0ebff2093 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T135102524536Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:378666ea1c3b68db99561409d0241f118ee763424db588991f355c005d8e4413 +size 6608518 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T135103906303Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T135103906303Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fcc4fd5767c5aa79b6cfdd03a50a45cebcf584fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T135103906303Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788781568, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 6482070528, + "inactive_split_bytes_peak": 8757777408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:03.905868Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788781568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 6660328448, + "inactive_split_bytes_peak": 8757777408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:02.646046Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6482070528, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 8757777408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107567120384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T135103906303Z.pickle", + "timestamp_utc": "2026-08-01T13:51:04.017413Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T135103906303Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T135103906303Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1e2f5acfc684feef2fe11974a04c31cdbc21cd8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T135103906303Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:384ef9729289f7d852b99ecf62fb5bf0664bfcbdffcabd336e9e3fac9763bf88 +size 6608773 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T135105298062Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T135105298062Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c307d0e2acbedcaa54349ba80d9141c8b4febc5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T135105298062Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788781568, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 7012649984, + "inactive_split_bytes_peak": 8757777408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:05.297622Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788781568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 6482070528, + "inactive_split_bytes_peak": 8757777408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:04.032213Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7012649984, + "inactive_split_bytes_current_delta": 530579456, + "inactive_split_bytes_peak": 8757777408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107567120384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T135105298062Z.pickle", + "timestamp_utc": "2026-08-01T13:51:05.408108Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T135105298062Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T135105298062Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e50b515fc170430c74456fc7cffe7b68190ec55c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T135105298062Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:519e774b19465867966b77f158388ceb07eb2fe1c6588d6b004d82aec5e6cdab +size 6607749 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T135106432625Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T135106432625Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4c6ae6551613599fd860b89a878f07b86691a871 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T135106432625Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788781568, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 6303812608, + "inactive_split_bytes_peak": 8757777408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:06.432225Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788781568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 7012649984, + "inactive_split_bytes_peak": 8757777408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:05.423203Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6303812608, + "inactive_split_bytes_current_delta": -708837376, + "inactive_split_bytes_peak": 8757777408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107567120384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T135106432625Z.pickle", + "timestamp_utc": "2026-08-01T13:51:06.540111Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T135106432625Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T135106432625Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e2dfee1b59732a74cacbe4506f9096c35a9e234 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T135106432625Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0100f226e68b455b5310b01e6e6b9317347a4093b5d838f7e4d16e0424b45384 +size 6607463 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T135107578611Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T135107578611Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc9ff5dad89686f79370e159f6ce0fcaf64f2ec8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T135107578611Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788781568, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 6777768960, + "inactive_split_bytes_peak": 8757777408, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:07.578204Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788781568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 6303812608, + "inactive_split_bytes_peak": 8757777408, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:06.554971Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6777768960, + "inactive_split_bytes_current_delta": 473956352, + "inactive_split_bytes_peak": 8757777408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107567120384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T135107578611Z.pickle", + "timestamp_utc": "2026-08-01T13:51:07.685394Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T135107578611Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T135107578611Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4d12f7d162bfefa7e6bc82fe0703016985f784db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T135107578611Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4722aab9e74ddb086bc22212463b1484a9fe761c075efe55d59d277855892ec9 +size 6607067 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T135108890698Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T135108890698Z.json new file mode 100644 index 0000000000000000000000000000000000000000..af21c8ad9739ad9e240c050124c301b331d2f3db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T135108890698Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788781568, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 6482070528, + "inactive_split_bytes_peak": 8757777408, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:08.890293Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788781568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 6777768960, + "inactive_split_bytes_peak": 8757777408, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:07.700112Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6482070528, + "inactive_split_bytes_current_delta": -295698432, + "inactive_split_bytes_peak": 8757777408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107567120384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T135108890698Z.pickle", + "timestamp_utc": "2026-08-01T13:51:08.995784Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T135108890698Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T135108890698Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..742c7d75c1cee6711f823e2b3ed38d3911dd07d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T135108890698Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00181893b0ee458def22f112cafad035ede1c88661a77bd128b339324998e4e5 +size 6607508 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T135110037372Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T135110037372Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6d65eedb8ff589b5a91f8f5833fe372cc1240f63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T135110037372Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788781568, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 7012649984, + "inactive_split_bytes_peak": 8757777408, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:10.036963Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788781568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 6482070528, + "inactive_split_bytes_peak": 8757777408, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:09.010545Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7012649984, + "inactive_split_bytes_current_delta": 530579456, + "inactive_split_bytes_peak": 8757777408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107567120384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T135110037372Z.pickle", + "timestamp_utc": "2026-08-01T13:51:10.147467Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T135110037372Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T135110037372Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6d59ceb3577ca80f16882a29e3b84c034fb352b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T135110037372Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b82404e80da4c917e86a9e2d3aec16d56c18ffba4b27093b48afde94b689c4ee +size 6607072 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T135205052119Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T135205052119Z.json new file mode 100644 index 0000000000000000000000000000000000000000..781ba394e50dd4f78a51aefdb0511b57402f31d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T135205052119Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105788781568, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 8652622848, + "inactive_split_bytes_peak": 8967137792, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:52:05.051653Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105788781568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105788781568, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788781568, + "device": 0, + "inactive_split_bytes_current": 7012649984, + "inactive_split_bytes_peak": 8757777408, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_peak": 107567120384, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T13:51:10.165002Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8652622848, + "inactive_split_bytes_current_delta": 1639972864, + "inactive_split_bytes_peak": 8967137792, + "inactive_split_bytes_peak_delta": 209360384, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107567120384, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107567120384, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T135205052119Z.pickle", + "timestamp_utc": "2026-08-01T13:52:05.253414Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T135205052119Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T135205052119Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d40c1334d8e3c243f7192715d091753714d945de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T135205052119Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f1ee5b702d1af844be8a60bd7aa269483cdd6218a6461d9a340af511e89e848 +size 9964875 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T135100155184Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T135100155184Z.json new file mode 100644 index 0000000000000000000000000000000000000000..72ce5fb336561865e849561d380db32b13ed5259 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T135100155184Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 82152062976, + "active_bytes_peak_delta": 46685726720, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82152062976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82152062976, + "device": 1, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_peak": 7448858112, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 94736744448, + "reserved_bytes_peak": 94736744448, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T13:51:00.154584Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 82152062976, + "allocated_bytes_peak_delta": 46685726720, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:49:52.072133Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_current_delta": 7316038656, + "inactive_split_bytes_peak": 7448858112, + "inactive_split_bytes_peak_delta": 7288472064, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 94736744448, + "reserved_bytes_current_delta": 55492739072, + "reserved_bytes_peak": 94736744448, + "reserved_bytes_peak_delta": 55492739072, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T135100155184Z.pickle", + "timestamp_utc": "2026-08-01T13:51:00.274494Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T135100155184Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T135100155184Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ee779a484cd09da858cd5b171973078f0242473 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T135100155184Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f895b0dee8cf47654664a29155fc44697fa2d3fdda8d9c71f4118e0ee6ba9473 +size 5923905 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T135101378454Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T135101378454Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f8a61b08b5d43d0dc8c0953bcbe046b2f49765d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T135101378454Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784420352, + "active_bytes_peak_delta": 23632357376, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 1, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_peak": 7451663872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107093164032, + "reserved_bytes_peak": 107093164032, + "segment_count_current": 479, + "segment_count_peak": 479, + "timestamp_utc": "2026-08-01T13:51:01.378032Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784420352, + "allocated_bytes_peak_delta": 23632357376, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82152062976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82152062976, + "device": 1, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_peak": 7448858112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 94736744448, + "reserved_bytes_peak": 94736744448, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T13:51:00.294298Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_current_delta": -645922816, + "inactive_split_bytes_peak": 7451663872, + "inactive_split_bytes_peak_delta": 2805760, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107093164032, + "reserved_bytes_current_delta": 12356419584, + "reserved_bytes_peak": 107093164032, + "reserved_bytes_peak_delta": 12356419584, + "segment_count_current": 479, + "segment_count_current_delta": 59, + "segment_count_peak": 479, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T135101378454Z.pickle", + "timestamp_utc": "2026-08-01T13:51:01.497687Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T135101378454Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T135101378454Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df2da4e4766b737a4f2dcb49793676c58618c6fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T135101378454Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:381dfed3f475549bc04cea077b64df781e948f197b71185f27df40b999ccdfd2 +size 5920101 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T135102524537Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T135102524537Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04e41c1e6f14cb4fa4a6638743a7b1e13e4f4cee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T135102524537Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784420352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 1, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_peak": 8354811904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:02.524137Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784420352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 1, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_peak": 7451663872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107093164032, + "reserved_bytes_peak": 107093164032, + "segment_count_current": 479, + "segment_count_peak": 479, + "timestamp_utc": "2026-08-01T13:51:01.515175Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 903148032, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 480, + "segment_count_current_delta": 1, + "segment_count_peak": 480, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T135102524537Z.pickle", + "timestamp_utc": "2026-08-01T13:51:02.636480Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T135102524537Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T135102524537Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f11539639eccb1597d099250786cd9a8335569bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T135102524537Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd76951b602885e8bcc3623db49f2467844f27a71b1a03aa483f1bf871b705fb +size 5937286 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T135103906352Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T135103906352Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5602b995331beb4a34b510ba8397e93919fe2f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T135103906352Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 2862592, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:03.905924Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 2862592, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 1, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_peak": 8354811904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:02.653628Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -408944640, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T135103906352Z.pickle", + "timestamp_utc": "2026-08-01T13:51:04.026604Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T135103906352Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T135103906352Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca100085131786c905382b729a076faaf5cd67f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T135103906352Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:397f9fafe4a15cc0119d8bc9fb9200e4d9ff1edbf5f18919792a4972ad5aaaa2 +size 5918065 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T135105298043Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T135105298043Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94b7745bc314580e566236e62751b866dddb829e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T135105298043Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:05.297616Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:04.045567Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T135105298043Z.pickle", + "timestamp_utc": "2026-08-01T13:51:05.409015Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T135105298043Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T135105298043Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44ce27bc6d2d78a20f67c81f99094ae8f39ba1f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T135105298043Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b167d5edcb96c9d6913cfe087929cb6dcbfb09230507b84ac9c8901c7493abf6 +size 5918058 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T135106432610Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T135106432610Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8bbac5ad1f4d4dd934b2aa162ef578e609ebc761 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T135106432610Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:06.432212Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:05.426005Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T135106432610Z.pickle", + "timestamp_utc": "2026-08-01T13:51:06.542843Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T135106432610Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T135106432610Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff1548bac1906a5e24541aeead06409a34617268 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T135106432610Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:505a48c6e5ee9438e95d7c6dcc9fa661f5a1dcb2867603e06d415ac22d91d7a0 +size 5918062 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T135107578638Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T135107578638Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61a9f8324b6354f42f1e6b6160a3a84b171b02ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T135107578638Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:07.578203Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:06.559248Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T135107578638Z.pickle", + "timestamp_utc": "2026-08-01T13:51:07.691097Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T135107578638Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T135107578638Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..26e8d4d9258f8325ddc659ec8bdb825d0ffdaf09 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T135107578638Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7b702b34bd86919b604c165cb8ac26c1fe76f5effcd4fba175ff256a9ddf993 +size 5918058 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T135108890686Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T135108890686Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d6f04bed31f3d103824d3a7537e4e03b3d76f5b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T135108890686Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:08.890275Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:07.707552Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T135108890686Z.pickle", + "timestamp_utc": "2026-08-01T13:51:09.000347Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T135108890686Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T135108890686Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a89d5504549547bbecdef0ff51d0541c613c8c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T135108890686Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5385f47dc763cc1a48cb0d73ed32f4ce3a33aedbb340aa2e5a4e3fd0449fa56b +size 5918067 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T135110037367Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T135110037367Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a46b9f21e676b48d4627fe5297af5196ddd115cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T135110037367Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:10.036954Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:09.016561Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T135110037367Z.pickle", + "timestamp_utc": "2026-08-01T13:51:10.146216Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T135110037367Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T135110037367Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f10a82f2b374db2705cd60825266b6fefc8c66bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T135110037367Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d820b572279221f8d06122a9d1e5770ce630041f7e55cc88413b097e0f29f397 +size 5918062 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T135205052125Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T135205052125Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5598ca9c456a8dc78376360849febc3220c7e8ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T135205052125Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 8858143744, + "inactive_split_bytes_peak": 9172658688, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:52:05.051657Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 1, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:10.163421Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8858143744, + "inactive_split_bytes_current_delta": 2170552320, + "inactive_split_bytes_peak": 9172658688, + "inactive_split_bytes_peak_delta": 817846784, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T135205052125Z.pickle", + "timestamp_utc": "2026-08-01T13:52:05.255582Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T135205052125Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T135205052125Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fded31e18c96a95fa6636ed80d0714fe928bdfd3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T135205052125Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b65f7ffe0513a018b5178bae79b9a00e3dfb7f7fd1e9ed164d6a1b7a328e7181 +size 9679112 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T135100155198Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T135100155198Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f85751cba2143b2d35384af48c4585b2888fa267 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T135100155198Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 82152062976, + "active_bytes_peak_delta": 46685726720, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82152062976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82152062976, + "device": 2, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_peak": 7448858112, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 94736744448, + "reserved_bytes_peak": 94736744448, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T13:51:00.154586Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 82152062976, + "allocated_bytes_peak_delta": 46685726720, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:49:52.072102Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_current_delta": 7316038656, + "inactive_split_bytes_peak": 7448858112, + "inactive_split_bytes_peak_delta": 7288472064, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 94736744448, + "reserved_bytes_current_delta": 55492739072, + "reserved_bytes_peak": 94736744448, + "reserved_bytes_peak_delta": 55492739072, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T135100155198Z.pickle", + "timestamp_utc": "2026-08-01T13:51:00.273290Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T135100155198Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T135100155198Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a7d06946d83f3bb2c5b7d8408007b9c17238a800 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T135100155198Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d1d63adc5b9d2829a3b71db7edd92578f99b6afd2d7b58b4c6c023a34477ea2 +size 5923909 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T135101378421Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T135101378421Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c84b80cf32091f1d192e9ef53f1a6f54b99ecec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T135101378421Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784420352, + "active_bytes_peak_delta": 23632357376, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 2, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_peak": 7451663872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107093164032, + "reserved_bytes_peak": 107093164032, + "segment_count_current": 479, + "segment_count_peak": 479, + "timestamp_utc": "2026-08-01T13:51:01.378001Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784420352, + "allocated_bytes_peak_delta": 23632357376, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82152062976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82152062976, + "device": 2, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_peak": 7448858112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 94736744448, + "reserved_bytes_peak": 94736744448, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T13:51:00.289565Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_current_delta": -645922816, + "inactive_split_bytes_peak": 7451663872, + "inactive_split_bytes_peak_delta": 2805760, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107093164032, + "reserved_bytes_current_delta": 12356419584, + "reserved_bytes_peak": 107093164032, + "reserved_bytes_peak_delta": 12356419584, + "segment_count_current": 479, + "segment_count_current_delta": 59, + "segment_count_peak": 479, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T135101378421Z.pickle", + "timestamp_utc": "2026-08-01T13:51:01.492950Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T135101378421Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T135101378421Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dad519c29d61a51f54357bcd8c7bec7680a4c8ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T135101378421Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97886d6ddcf004bdd830e29ec4f3b17b3a91adbe28bce4f39fba2d9a67bfab8f +size 5920096 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T135102524498Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T135102524498Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61b2f4327bed1f022afb23ec9363ffb9a4c048a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T135102524498Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784420352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 2, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_peak": 8354811904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:02.524101Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784420352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 2, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_peak": 7451663872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107093164032, + "reserved_bytes_peak": 107093164032, + "segment_count_current": 479, + "segment_count_peak": 479, + "timestamp_utc": "2026-08-01T13:51:01.505666Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 903148032, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 480, + "segment_count_current_delta": 1, + "segment_count_peak": 480, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T135102524498Z.pickle", + "timestamp_utc": "2026-08-01T13:51:02.625438Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T135102524498Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T135102524498Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7d4098f4543ca7cbc4ff54ce52f5727379c819b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T135102524498Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfe8ac74e00e1d0d5d5d444b66a13ce9b0368cd65539f35346abeac8066f13a3 +size 5937286 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T135103906230Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T135103906230Z.json new file mode 100644 index 0000000000000000000000000000000000000000..749f74164ecac382e9aabed83f0d81920a8f2bae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T135103906230Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 2862592, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:03.905813Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 2862592, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 2, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_peak": 8354811904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:02.637124Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -408944640, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T135103906230Z.pickle", + "timestamp_utc": "2026-08-01T13:51:04.008325Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T135103906230Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T135103906230Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..77815dcd03c97d670f1e5a6e6ccf597828326818 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T135103906230Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e521e0e048cb88b30602faffff03756fe08ff3dd94c6b3cbf6ec76a3e8776e8f +size 5918066 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T135105298011Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T135105298011Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2bdea0a5051a25d9750495c7ac3eeda5273010f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T135105298011Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:05.297590Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:04.019888Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T135105298011Z.pickle", + "timestamp_utc": "2026-08-01T13:51:05.405537Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T135105298011Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T135105298011Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f13291ab51d5577f2f8423835758b859872bc2a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T135105298011Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff12c91aca29d67d84ce2ff1b3a005175d5bc8ee8cd49088e72c7123950aac90 +size 5918058 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T135106432575Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T135106432575Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b2bb3158ee743f7d0dbf882530972348ddd42c83 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T135106432575Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:06.432168Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:05.417772Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T135106432575Z.pickle", + "timestamp_utc": "2026-08-01T13:51:06.532276Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T135106432575Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T135106432575Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13f032c35aa982289b3f27f837d774476ea9df4e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T135106432575Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2530133d90dab86898e1f994b2df2d74851b75be7458b7f4ddbd708cc9275c0d +size 5918073 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T135107578595Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T135107578595Z.json new file mode 100644 index 0000000000000000000000000000000000000000..714358db235090f8705d1a40a8a291367a73c3c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T135107578595Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:07.578166Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:06.543728Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T135107578595Z.pickle", + "timestamp_utc": "2026-08-01T13:51:07.678078Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T135107578595Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T135107578595Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf028d50498bb1287b13248a3207fbedaa2db8ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T135107578595Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c8d21fbf1efac38718816d78b47aede3e2e8fb62bb9cae0b76fc8150c61f181 +size 5918061 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T135108890636Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T135108890636Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb6ee683700efb93ae01272f0f7db3c64bedc94b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T135108890636Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:08.890246Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:07.689563Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T135108890636Z.pickle", + "timestamp_utc": "2026-08-01T13:51:08.990453Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T135108890636Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T135108890636Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..520820f752a5ec61d895057a6660a8f63f91c4e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T135108890636Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82815c9ee361ec221b20cb4005efb0936748d677bd026e7453937dad4651d62f +size 5918072 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T135110037355Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T135110037355Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3578aa9e953a558c54f2cb9cb17d6dc8abbb1b51 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T135110037355Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:10.036938Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:09.001846Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T135110037355Z.pickle", + "timestamp_utc": "2026-08-01T13:51:10.144946Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T135110037355Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T135110037355Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..83b4d3742892a9af1b51d347e21f3339e383ba8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T135110037355Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a794da9701996fc375027715ca7d84b11ee815900d48e8de065a938c8c0f0d0c +size 5918054 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T135205052135Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T135205052135Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea426cb1ac6c99324bfc98d72f6be23cea639edd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T135205052135Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 8858143744, + "inactive_split_bytes_peak": 9172658688, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:52:05.051655Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 2, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:10.158555Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8858143744, + "inactive_split_bytes_current_delta": 2170552320, + "inactive_split_bytes_peak": 9172658688, + "inactive_split_bytes_peak_delta": 817846784, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T135205052135Z.pickle", + "timestamp_utc": "2026-08-01T13:52:05.263047Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T135205052135Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T135205052135Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8fb5baec6ea9cbc16d13ce358feb59302660a03f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T135205052135Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e3ff2b1ecd32afd5d4efea7286de2063fe76df15b931c5c15b1ee24929c77af +size 9679147 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T135100155393Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T135100155393Z.json new file mode 100644 index 0000000000000000000000000000000000000000..935a09a10ced06af7fcca8f9df3041579e924efd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T135100155393Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 82152062976, + "active_bytes_peak_delta": 46685726720, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82152062976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82152062976, + "device": 3, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_peak": 7448858112, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 94736744448, + "reserved_bytes_peak": 94736744448, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T13:51:00.154769Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 82152062976, + "allocated_bytes_peak_delta": 46685726720, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:49:52.072116Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_current_delta": 7316038656, + "inactive_split_bytes_peak": 7448858112, + "inactive_split_bytes_peak_delta": 7288472064, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 94736744448, + "reserved_bytes_current_delta": 55492739072, + "reserved_bytes_peak": 94736744448, + "reserved_bytes_peak_delta": 55492739072, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T135100155393Z.pickle", + "timestamp_utc": "2026-08-01T13:51:00.280739Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T135100155393Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T135100155393Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dc5a30a4f5fe417993018a640fc48b171bf0a134 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T135100155393Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:223a4698faca12f4c550312348ea0fd04dda439e51d17d3bc8dfed1369b0038c +size 5923910 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T135101378494Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T135101378494Z.json new file mode 100644 index 0000000000000000000000000000000000000000..44da9ca1888afbda29161a78fcb006433abae073 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T135101378494Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784420352, + "active_bytes_peak_delta": 23632357376, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 3, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_peak": 7451663872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107093164032, + "reserved_bytes_peak": 107093164032, + "segment_count_current": 479, + "segment_count_peak": 479, + "timestamp_utc": "2026-08-01T13:51:01.378040Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784420352, + "allocated_bytes_peak_delta": 23632357376, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82152062976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82152062976, + "device": 3, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_peak": 7448858112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 94736744448, + "reserved_bytes_peak": 94736744448, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T13:51:00.301128Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_current_delta": -645922816, + "inactive_split_bytes_peak": 7451663872, + "inactive_split_bytes_peak_delta": 2805760, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107093164032, + "reserved_bytes_current_delta": 12356419584, + "reserved_bytes_peak": 107093164032, + "reserved_bytes_peak_delta": 12356419584, + "segment_count_current": 479, + "segment_count_current_delta": 59, + "segment_count_peak": 479, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T135101378494Z.pickle", + "timestamp_utc": "2026-08-01T13:51:01.497296Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T135101378494Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T135101378494Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e01f17cb9303fdae273b8c6ec9628e84ecd9aa9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T135101378494Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67cd0826d3cb86d2648cfa3cfc2514fd44645db48717649331598f7af6114cbc +size 5920095 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T135102524552Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T135102524552Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6d98bb5dbd8bd5fc323e61cc4d362783ae36e4c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T135102524552Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784420352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 3, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_peak": 8354811904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:02.524145Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784420352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 3, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_peak": 7451663872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107093164032, + "reserved_bytes_peak": 107093164032, + "segment_count_current": 479, + "segment_count_peak": 479, + "timestamp_utc": "2026-08-01T13:51:01.514906Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 903148032, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 480, + "segment_count_current_delta": 1, + "segment_count_peak": 480, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T135102524552Z.pickle", + "timestamp_utc": "2026-08-01T13:51:02.635920Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T135102524552Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T135102524552Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f10dde61da3df6847a085645094e29bbfd50557 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T135102524552Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4da9c0beea179d48b6c792ed884f0405a6349d3feec7ec363253de1468f07b7 +size 5937282 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T135103906291Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T135103906291Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd32d4903c141f4b69adabfe0cefbe33272b1e93 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T135103906291Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 2862592, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:03.905863Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 2862592, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 3, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_peak": 8354811904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:02.653124Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -408944640, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T135103906291Z.pickle", + "timestamp_utc": "2026-08-01T13:51:04.017620Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T135103906291Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T135103906291Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..01b7a5405bc53740cc9211fd22e2a7fd4c72a8a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T135103906291Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e790cae009befca3567fc364dc09d392375ffb461d146d0cc5663cfb0954fe57 +size 5918098 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T135105298039Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T135105298039Z.json new file mode 100644 index 0000000000000000000000000000000000000000..698a06c98907cbd2c66cfb639faa71e074e95ebb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T135105298039Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:05.297622Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:04.034026Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T135105298039Z.pickle", + "timestamp_utc": "2026-08-01T13:51:05.408701Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T135105298039Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T135105298039Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c795af8b1ad6f195ede1160de52b6f3f842cb253 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T135105298039Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfa08366a22df637768179a4e70f03c7f21619a76e8d6e1ffb813c377dde2ef3 +size 5918064 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T135106432644Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T135106432644Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cdcf6f86684030db9cfa0367ee04f4e855a210ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T135106432644Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:06.432229Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:05.425338Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T135106432644Z.pickle", + "timestamp_utc": "2026-08-01T13:51:06.548238Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T135106432644Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T135106432644Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c1f0b61e7a6f29d3dba9ad03353f04a247865e86 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T135106432644Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab7ad5a9ae7bc2e4491f6701d0dc4ac55abd7a786528020cb984d8967a0c8e90 +size 5918096 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T135107578593Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T135107578593Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1642c59e83be3ce71ea3c87a98a2971b9b36f2f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T135107578593Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:07.578200Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:06.567166Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T135107578593Z.pickle", + "timestamp_utc": "2026-08-01T13:51:07.686417Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T135107578593Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T135107578593Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..186505bab8644543fb85147149ea0a1f3cff9686 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T135107578593Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d5e158cd3e182472717af67d4e49e7df4e3c810a38c339003df08558d4d4e9c +size 5918061 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T135108890709Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T135108890709Z.json new file mode 100644 index 0000000000000000000000000000000000000000..13bfdc08db4b89a588e3b13ddcf0a1639af52c33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T135108890709Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:08.890291Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:07.702576Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T135108890709Z.pickle", + "timestamp_utc": "2026-08-01T13:51:08.998569Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T135108890709Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T135108890709Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60be78efd6d0325f078d4e462d93fcb6f4952a5b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T135108890709Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84d9368582e3cffa67dd282da2cab0494483be35de8499bef7afbd180740c15b +size 5918097 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T135110037392Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T135110037392Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f9fb28a46ed31f085493d952eb39c42eb06b5a38 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T135110037392Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:10.036984Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:09.014894Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T135110037392Z.pickle", + "timestamp_utc": "2026-08-01T13:51:10.144811Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T135110037392Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T135110037392Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1bd92fe50a6e75064f1e33dab6f0864b7d7e214 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T135110037392Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c74947cbfe80f3b7e59e6160f11bd8e84dfd0efbc0756a0b254efbd67b034408 +size 5918064 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T135205052255Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T135205052255Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3c770b3af099c36a86c155ddb01dd2ccb8acc196 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T135205052255Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 8858143744, + "inactive_split_bytes_peak": 9172658688, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:52:05.051674Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 3, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:10.162038Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8858143744, + "inactive_split_bytes_current_delta": 2170552320, + "inactive_split_bytes_peak": 9172658688, + "inactive_split_bytes_peak_delta": 817846784, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T135205052255Z.pickle", + "timestamp_utc": "2026-08-01T13:52:05.264748Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T135205052255Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T135205052255Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1946c12b7be9001819921e1c954ec761ca658004 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T135205052255Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53f9b44ca6baf114eb19c4d015ee18a344372ef59149dbe7c552bef3553e2070 +size 9679143 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T135100155170Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T135100155170Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a183fb3cc8fcc9893dc1a5107075a086d8f58b57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T135100155170Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441708544, + "active_bytes_current_delta": 37209810944, + "active_bytes_peak": 82154291200, + "active_bytes_peak_delta": 46687512576, + "after": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 82154291200, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 82154291200, + "device": 4, + "inactive_split_bytes_current": 6742556672, + "inactive_split_bytes_peak": 7083527168, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 94176804864, + "reserved_bytes_peak": 94176804864, + "segment_count_current": 417, + "segment_count_peak": 417, + "timestamp_utc": "2026-08-01T13:51:00.154573Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441708544, + "allocated_bytes_current_delta": 37209810944, + "allocated_bytes_peak": 82154291200, + "allocated_bytes_peak_delta": 46687512576, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:49:52.071868Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6742556672, + "inactive_split_bytes_current_delta": 6610180096, + "inactive_split_bytes_peak": 7083527168, + "inactive_split_bytes_peak_delta": 6930117632, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 94176804864, + "reserved_bytes_current_delta": 54932799488, + "reserved_bytes_peak": 94176804864, + "reserved_bytes_peak_delta": 54932799488, + "segment_count_current": 417, + "segment_count_current_delta": 346, + "segment_count_peak": 417, + "segment_count_peak_delta": 346, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T135100155170Z.pickle", + "timestamp_utc": "2026-08-01T13:51:00.271872Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T135100155170Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T135100155170Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a75444392aca8cffd7bbf8634570ea35a6723b25 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T135100155170Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc5ee9122f54dc48d461cdd23755044ec4c20d3bee1ebf6c4300ae8feab10e95 +size 5931956 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T135101378405Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T135101378405Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a2e36ee40099587f8629ae2c3e57d82d4ae8ec3b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T135101378405Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441708544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784419328, + "active_bytes_peak_delta": 23630128128, + "after": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105784419328, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105784419328, + "device": 4, + "inactive_split_bytes_current": 6744653824, + "inactive_split_bytes_peak": 8065564672, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107082678272, + "reserved_bytes_peak": 107082678272, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:01.377980Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441708544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784419328, + "allocated_bytes_peak_delta": 23630128128, + "before": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 82154291200, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 82154291200, + "device": 4, + "inactive_split_bytes_current": 6742556672, + "inactive_split_bytes_peak": 7083527168, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 94176804864, + "reserved_bytes_peak": 94176804864, + "segment_count_current": 417, + "segment_count_peak": 417, + "timestamp_utc": "2026-08-01T13:51:00.293331Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6744653824, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 8065564672, + "inactive_split_bytes_peak_delta": 982037504, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107082678272, + "reserved_bytes_current_delta": 12905873408, + "reserved_bytes_peak": 107082678272, + "reserved_bytes_peak_delta": 12905873408, + "segment_count_current": 480, + "segment_count_current_delta": 63, + "segment_count_peak": 480, + "segment_count_peak_delta": 63, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T135101378405Z.pickle", + "timestamp_utc": "2026-08-01T13:51:01.495437Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T135101378405Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T135101378405Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d15b2fe60adff650d70e2c2c975ab781128a64e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T135101378405Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:749678b35f936a02741fdc09714b8c9c304e41d23f710cc08f5cb890c891b639 +size 6073233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T135102524474Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T135102524474Z.json new file mode 100644 index 0000000000000000000000000000000000000000..856549477ff0f79f5d18fb9b41f02f83bca14c4c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T135102524474Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441708544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788220928, + "active_bytes_peak_delta": 3801600, + "after": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 7153598464, + "inactive_split_bytes_peak": 8065564672, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:02.524065Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441708544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788220928, + "allocated_bytes_peak_delta": 3801600, + "before": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105784419328, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105784419328, + "device": 4, + "inactive_split_bytes_current": 6744653824, + "inactive_split_bytes_peak": 8065564672, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107082678272, + "reserved_bytes_peak": 107082678272, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:01.512389Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7153598464, + "inactive_split_bytes_current_delta": 408944640, + "inactive_split_bytes_peak": 8065564672, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107552440320, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107552440320, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 481, + "segment_count_current_delta": 1, + "segment_count_peak": 481, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T135102524474Z.pickle", + "timestamp_utc": "2026-08-01T13:51:02.639855Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T135102524474Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T135102524474Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c8778b24c1e65c92c923a86d1f0cabb8f17ab4f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T135102524474Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f142507473f0b528004cdf4863e69ed63264a90d42e6b2be00324ffdf2bf604e +size 5940658 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T135103906222Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T135103906222Z.json new file mode 100644 index 0000000000000000000000000000000000000000..58fcbc3e027d5aeaff456c064fbc14e6da97f929 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T135103906222Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441708544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788220928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 6922911744, + "inactive_split_bytes_peak": 8308738560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:03.905802Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441708544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788220928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 7153598464, + "inactive_split_bytes_peak": 8065564672, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:02.658468Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6922911744, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 8308738560, + "inactive_split_bytes_peak_delta": 243173888, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107552440320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107552440320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 481, + "segment_count_current_delta": 0, + "segment_count_peak": 481, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T135103906222Z.pickle", + "timestamp_utc": "2026-08-01T13:51:04.019877Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T135103906222Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T135103906222Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b3010c144ea47baa5be7fd04cc3b23b94076b557 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T135103906222Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf5f265f3dc19266a433d07ea5879e0c62982cf8facea29ecf425dfe9a444640 +size 6059282 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T135105298007Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T135105298007Z.json new file mode 100644 index 0000000000000000000000000000000000000000..82b609ae06a08f20ad7e68d63794ac15fa69f85c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T135105298007Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441708544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788220928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 7271038976, + "inactive_split_bytes_peak": 8561147904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:05.297565Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441708544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788220928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 6922911744, + "inactive_split_bytes_peak": 8308738560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:04.036188Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7271038976, + "inactive_split_bytes_current_delta": 348127232, + "inactive_split_bytes_peak": 8561147904, + "inactive_split_bytes_peak_delta": 252409344, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107552440320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107552440320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 481, + "segment_count_current_delta": 0, + "segment_count_peak": 481, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T135105298007Z.pickle", + "timestamp_utc": "2026-08-01T13:51:05.410899Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T135105298007Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T135105298007Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0dcb93d62b2fc015a60bb99cafb309b413bd453f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T135105298007Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62e76a68e1be97ff50ca5e7393e0e7ac4f53b50521606797d709793e45b39450 +size 5939782 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T135106432578Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T135106432578Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f0e6d1734bba55f2bcccc343d6fc0f6cf15b30e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T135106432578Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441708544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788220928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 6922911744, + "inactive_split_bytes_peak": 8561147904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:06.432169Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441708544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788220928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 7271038976, + "inactive_split_bytes_peak": 8561147904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:05.427206Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6922911744, + "inactive_split_bytes_current_delta": -348127232, + "inactive_split_bytes_peak": 8561147904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107552440320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107552440320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 481, + "segment_count_current_delta": 0, + "segment_count_peak": 481, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T135106432578Z.pickle", + "timestamp_utc": "2026-08-01T13:51:06.541868Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T135106432578Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T135106432578Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0792e797c3a0c5ce88720b3971e5a95185823e1e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T135106432578Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:258de46829bdea6c34f434cbb9850f5a0345eccc80740fe6304cc57e97a7d019 +size 6059282 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T135107578569Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T135107578569Z.json new file mode 100644 index 0000000000000000000000000000000000000000..259ce97738a4e70bdf5b5702936af75460f524db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T135107578569Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441708544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788220928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 7271038976, + "inactive_split_bytes_peak": 8561147904, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:07.578165Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441708544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788220928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 6922911744, + "inactive_split_bytes_peak": 8561147904, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:06.557999Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7271038976, + "inactive_split_bytes_current_delta": 348127232, + "inactive_split_bytes_peak": 8561147904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107552440320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107552440320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 481, + "segment_count_current_delta": 0, + "segment_count_peak": 481, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T135107578569Z.pickle", + "timestamp_utc": "2026-08-01T13:51:07.687932Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T135107578569Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T135107578569Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80e068102a08669f6cb550c6b51ba4e99cc06df3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T135107578569Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c9964c5bbf9f5b709439edcae78b261077a045e9cf06ac815a111b80761b0a3 +size 5939785 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T135108890688Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T135108890688Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4bb6b73180ae2c9b31b5d473a227f24c3b4cf144 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T135108890688Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441708544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788220928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 6922911744, + "inactive_split_bytes_peak": 8561147904, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:08.890251Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441708544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788220928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 7271038976, + "inactive_split_bytes_peak": 8561147904, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:07.704074Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6922911744, + "inactive_split_bytes_current_delta": -348127232, + "inactive_split_bytes_peak": 8561147904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107552440320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107552440320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 481, + "segment_count_current_delta": 0, + "segment_count_peak": 481, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T135108890688Z.pickle", + "timestamp_utc": "2026-08-01T13:51:09.002122Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T135108890688Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T135108890688Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d604063c4401f96c698b910b3f4056310aefbe73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T135108890688Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe1b450d0f96d5893f052751d17461fc3b29dcdeb60cd9efe2050dee1931c3f2 +size 6059282 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T135110037388Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T135110037388Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fe975426b34d250b77b273d021afb298ac2bd3ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T135110037388Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441708544, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105788220928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 7271038976, + "inactive_split_bytes_peak": 8561147904, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:10.036953Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441708544, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788220928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 6922911744, + "inactive_split_bytes_peak": 8561147904, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:09.018553Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7271038976, + "inactive_split_bytes_current_delta": 348127232, + "inactive_split_bytes_peak": 8561147904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107552440320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107552440320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 481, + "segment_count_current_delta": 0, + "segment_count_peak": 481, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T135110037388Z.pickle", + "timestamp_utc": "2026-08-01T13:51:10.145900Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T135110037388Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T135110037388Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cbc7682eaa4ec05418a6c8c7f576d5649a89d696 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T135110037388Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e92bb91162290b056ccac32f6c7f393c871ca0801bffbb9a2ce5d02fc39be98a +size 5939779 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T135205052280Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T135205052280Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f464564af6fb9957338e985db6139cb9f6df30e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T135205052280Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61167419392, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105788220928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61167419392, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 61167419392, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 8328003584, + "inactive_split_bytes_peak": 8667684352, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:52:05.051681Z", + "world_size": 8 + }, + "allocated_bytes_current": 61167419392, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105788220928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441708544, + "active_bytes_peak": 105788220928, + "allocated_bytes_current": 72441708544, + "allocated_bytes_peak": 105788220928, + "device": 4, + "inactive_split_bytes_current": 7271038976, + "inactive_split_bytes_peak": 8561147904, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107552440320, + "reserved_bytes_peak": 107552440320, + "segment_count_current": 481, + "segment_count_peak": 481, + "timestamp_utc": "2026-08-01T13:51:10.162339Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8328003584, + "inactive_split_bytes_current_delta": 1056964608, + "inactive_split_bytes_peak": 8667684352, + "inactive_split_bytes_peak_delta": 106536448, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107552440320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107552440320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 481, + "segment_count_current_delta": 0, + "segment_count_peak": 481, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T135205052280Z.pickle", + "timestamp_utc": "2026-08-01T13:52:05.273926Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T135205052280Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T135205052280Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..feb63a3d560dcccda41e56213815e8f6a033fa85 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T135205052280Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1847497095f4224850db5e799f477c58994be83eac630d9720b0ddec6926a99c +size 9693339 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T135100155168Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T135100155168Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d9754b72fdcd7108638c9e96b897be1353d1bb9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T135100155168Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 82152062976, + "active_bytes_peak_delta": 46685726720, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82152062976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82152062976, + "device": 5, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_peak": 7448858112, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 94736744448, + "reserved_bytes_peak": 94736744448, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T13:51:00.154573Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 82152062976, + "allocated_bytes_peak_delta": 46685726720, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:49:52.072060Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_current_delta": 7316038656, + "inactive_split_bytes_peak": 7448858112, + "inactive_split_bytes_peak_delta": 7288472064, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 94736744448, + "reserved_bytes_current_delta": 55492739072, + "reserved_bytes_peak": 94736744448, + "reserved_bytes_peak_delta": 55492739072, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T135100155168Z.pickle", + "timestamp_utc": "2026-08-01T13:51:00.272614Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T135100155168Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T135100155168Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..46b0d45c74b0e330bb05eb552d219add782331ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T135100155168Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e665b03e0e2a8e7010745c51e282bb8f06561f6385391d59f231c877090f36e +size 5923910 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T135101378418Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T135101378418Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b9022e487b5f2bdb677e7fce34bf8f1c02e26b5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T135101378418Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784420352, + "active_bytes_peak_delta": 23632357376, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 5, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_peak": 7451663872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107093164032, + "reserved_bytes_peak": 107093164032, + "segment_count_current": 479, + "segment_count_peak": 479, + "timestamp_utc": "2026-08-01T13:51:01.378002Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784420352, + "allocated_bytes_peak_delta": 23632357376, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82152062976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82152062976, + "device": 5, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_peak": 7448858112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 94736744448, + "reserved_bytes_peak": 94736744448, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T13:51:00.289586Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_current_delta": -645922816, + "inactive_split_bytes_peak": 7451663872, + "inactive_split_bytes_peak_delta": 2805760, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107093164032, + "reserved_bytes_current_delta": 12356419584, + "reserved_bytes_peak": 107093164032, + "reserved_bytes_peak_delta": 12356419584, + "segment_count_current": 479, + "segment_count_current_delta": 59, + "segment_count_peak": 479, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T135101378418Z.pickle", + "timestamp_utc": "2026-08-01T13:51:01.494649Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T135101378418Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T135101378418Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf1d2fda4f9d7c8c1e35faf886af5b6ae4804d02 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T135101378418Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7771d6dfaa6c4db6c44d4aee704c2bf83cbcaa04095afe44112e93571b04c57b +size 5920097 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T135102524493Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T135102524493Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3fcebf4d2ee19a88fe330fb809cc719e721bc2fe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T135102524493Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784420352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 5, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_peak": 8354811904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:02.524092Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784420352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 5, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_peak": 7451663872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107093164032, + "reserved_bytes_peak": 107093164032, + "segment_count_current": 479, + "segment_count_peak": 479, + "timestamp_utc": "2026-08-01T13:51:01.510754Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 903148032, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 480, + "segment_count_current_delta": 1, + "segment_count_peak": 480, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T135102524493Z.pickle", + "timestamp_utc": "2026-08-01T13:51:02.631061Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T135102524493Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T135102524493Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..81c3ce0252cc70a1d3edcac2d04629188683546d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T135102524493Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3be648a9a20f2d11c9de53cb01d2e1a4084ab1376653d593d0f823a0629bcf82 +size 5937281 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T135103906222Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T135103906222Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5a8b4d98a81cf142c446fc186b356937714ac25 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T135103906222Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 2862592, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:03.905800Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 2862592, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 5, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_peak": 8354811904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:02.644613Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -408944640, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T135103906222Z.pickle", + "timestamp_utc": "2026-08-01T13:51:04.016766Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T135103906222Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T135103906222Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..39d341b67592db32de18a2f06d365ccfdf3c50a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T135103906222Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8813cc2a6f8896b75ca389f0552059cd16987045d51e9c070aee517c901ada20 +size 5918066 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T135105297971Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T135105297971Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bdaeacc815047280451e176003ada47447240671 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T135105297971Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:05.297565Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:04.031556Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T135105297971Z.pickle", + "timestamp_utc": "2026-08-01T13:51:05.408333Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T135105297971Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T135105297971Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da02e351c24024d69935f1cbd29288852cc18def --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T135105297971Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4e68f1ce9abade6019e917966a5ab3df315e400e02cf0f5875d1f76caf26e32 +size 5918058 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T135106432552Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T135106432552Z.json new file mode 100644 index 0000000000000000000000000000000000000000..194dc11889eb3ba93d9178ed027fc196abaf93fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T135106432552Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:06.432157Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:05.423016Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T135106432552Z.pickle", + "timestamp_utc": "2026-08-01T13:51:06.537073Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T135106432552Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T135106432552Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bb6f23864b768133bf6703360206f49bb54ce273 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T135106432552Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b825ca969a7347d06398244d98e7bdea9f375a0f033a1b6d4cd90367ca7666c +size 5918066 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T135107578570Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T135107578570Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06f010dd016c2992800a2c748553a1f5dcaaac78 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T135107578570Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:07.578166Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:06.551685Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T135107578570Z.pickle", + "timestamp_utc": "2026-08-01T13:51:07.684583Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T135107578570Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T135107578570Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7d87579cd9b39752736afa7d0ae5783265d40620 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T135107578570Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ddabca565ec2fddd555d543a063931ea74d6836114e1d3a87a3789b1e948617 +size 5918060 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T135108890638Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T135108890638Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70acfa7522304cea0024a63a41266958fc39cf39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T135108890638Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:08.890241Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:07.699267Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T135108890638Z.pickle", + "timestamp_utc": "2026-08-01T13:51:08.998179Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T135108890638Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T135108890638Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ee072c59cdfc083a30f7f2c4f39bdc7fe017a1e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T135108890638Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:923cee532df9138cedb3704c6e72b4f5be93e105512a30af1238384b61b93db3 +size 5918066 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T135110037340Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T135110037340Z.json new file mode 100644 index 0000000000000000000000000000000000000000..122b1c9a1c109fd7a93fc3bf2c59ccd8cd9d77ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T135110037340Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:10.036935Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:09.012811Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T135110037340Z.pickle", + "timestamp_utc": "2026-08-01T13:51:10.143449Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T135110037340Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T135110037340Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ade11127d47a5abd8b925b13ca6cfd866188c37a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T135110037340Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82df396c35338fc039b195b29458846174500a5ccedc530c232d554f95aea4a4 +size 5918056 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T135205052123Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T135205052123Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4973386f16fd51d986e3937f72c38df3fe6b8a56 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T135205052123Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 8858143744, + "inactive_split_bytes_peak": 9172658688, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:52:05.051662Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 5, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:10.157936Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8858143744, + "inactive_split_bytes_current_delta": 2170552320, + "inactive_split_bytes_peak": 9172658688, + "inactive_split_bytes_peak_delta": 817846784, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T135205052123Z.pickle", + "timestamp_utc": "2026-08-01T13:52:05.253104Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T135205052123Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T135205052123Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2924d799768f37bb67a6bbef4d6bd76978567f9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T135205052123Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d712b97eac8abad479c7841a2a01badecbfaf254bf2de2b1664e5b2307f7ad51 +size 9679112 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T135100155358Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T135100155358Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34477c59f6955b7441bd597caf2861cfd376efab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T135100155358Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 82152062976, + "active_bytes_peak_delta": 46685726720, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82152062976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82152062976, + "device": 6, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_peak": 7448858112, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 94736744448, + "reserved_bytes_peak": 94736744448, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T13:51:00.154743Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 82152062976, + "allocated_bytes_peak_delta": 46685726720, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:49:52.072114Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_current_delta": 7316038656, + "inactive_split_bytes_peak": 7448858112, + "inactive_split_bytes_peak_delta": 7288472064, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 94736744448, + "reserved_bytes_current_delta": 55492739072, + "reserved_bytes_peak": 94736744448, + "reserved_bytes_peak_delta": 55492739072, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T135100155358Z.pickle", + "timestamp_utc": "2026-08-01T13:51:00.273399Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T135100155358Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T135100155358Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1f9e0d5a54219f3f0ef3e68b94f6d4b96f8db06d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T135100155358Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:772c65fba6b386c0b45c7530c84c3cccd3e4a2994c05c51b847196316bcd7494 +size 5923908 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T135101378434Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T135101378434Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ad90067f44d5b8043d3046be97d397257cc316d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T135101378434Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784420352, + "active_bytes_peak_delta": 23632357376, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 6, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_peak": 7451663872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107093164032, + "reserved_bytes_peak": 107093164032, + "segment_count_current": 479, + "segment_count_peak": 479, + "timestamp_utc": "2026-08-01T13:51:01.378014Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784420352, + "allocated_bytes_peak_delta": 23632357376, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82152062976, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82152062976, + "device": 6, + "inactive_split_bytes_current": 7448857600, + "inactive_split_bytes_peak": 7448858112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 94736744448, + "reserved_bytes_peak": 94736744448, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T13:51:00.289094Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_current_delta": -645922816, + "inactive_split_bytes_peak": 7451663872, + "inactive_split_bytes_peak_delta": 2805760, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107093164032, + "reserved_bytes_current_delta": 12356419584, + "reserved_bytes_peak": 107093164032, + "reserved_bytes_peak_delta": 12356419584, + "segment_count_current": 479, + "segment_count_current_delta": 59, + "segment_count_peak": 479, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T135101378434Z.pickle", + "timestamp_utc": "2026-08-01T13:51:01.491241Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T135101378434Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T135101378434Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5651c5c6d4bc4b2167b90e082221a2179af8f246 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T135101378434Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43234b2255a7dd6bb85c3514b6ea0ae8e9c8c684f37b2e5a7a5915559a0ba701 +size 5920100 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T135102524499Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T135102524499Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f7208fcd491e56efd22e6b0e0bc060d4feaec61 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T135102524499Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784420352, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 6, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_peak": 8354811904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:02.524101Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784420352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 6, + "inactive_split_bytes_current": 6802934784, + "inactive_split_bytes_peak": 7451663872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107093164032, + "reserved_bytes_peak": 107093164032, + "segment_count_current": 479, + "segment_count_peak": 479, + "timestamp_utc": "2026-08-01T13:51:01.503602Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 903148032, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 480, + "segment_count_current_delta": 1, + "segment_count_peak": 480, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T135102524499Z.pickle", + "timestamp_utc": "2026-08-01T13:51:02.624124Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T135102524499Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T135102524499Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72edd7532efcf1c5276ff9995a8c1f8682a88969 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T135102524499Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc9fd30746faa9b5f21abf524de33f9c48be9abc866e429d2f490b083b86927f +size 5937286 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T135103906229Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T135103906229Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d33f414aa7106f580781fc2df90a33632215a375 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T135103906229Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 2862592, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:03.905820Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 2862592, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105784420352, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784420352, + "device": 6, + "inactive_split_bytes_current": 6626774016, + "inactive_split_bytes_peak": 8354811904, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:02.635522Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -408944640, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T135103906229Z.pickle", + "timestamp_utc": "2026-08-01T13:51:04.004971Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T135103906229Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T135103906229Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76c3b2af2eaf6dd5c0f4f620e5d249fdce176ce3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T135103906229Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1f3f2dd39ec7feb5ef7efde68b675fcee44945174262cc3f700c80263415f10 +size 5918068 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T135105298007Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T135105298007Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e52969654a7477eff44dbbc64687ab79c809fba8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T135105298007Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:05.297593Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:04.016366Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T135105298007Z.pickle", + "timestamp_utc": "2026-08-01T13:51:05.401952Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T135105298007Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T135105298007Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ce2a01edac23338cdf50f95900fb46325703232 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T135105298007Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:126a97a17629de684ba06bdd9568af9a134aba8a361bd03d058607f40f3fe0d5 +size 5918058 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T135106432578Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T135106432578Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c258add7831798828c45daa4949c0c86c7a2378b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T135106432578Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:06.432188Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:05.413595Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T135106432578Z.pickle", + "timestamp_utc": "2026-08-01T13:51:06.527465Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T135106432578Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T135106432578Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d70dcddaf5ad7ecfd1f5a433d7a17bcd2dc52a71 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T135106432578Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33d73fc2419a2c3bc2ddd83498f913956360e5d9bdaf4ba78e6143d6d614c451 +size 5918065 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T135107578603Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T135107578603Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e435f2de64a6eefaf38267902b6f59f289e6f79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T135107578603Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:07.578204Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:06.538557Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T135107578603Z.pickle", + "timestamp_utc": "2026-08-01T13:51:07.674315Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T135107578603Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T135107578603Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cca14c1abe7a1ab239059f8cc3ac8544b61e2802 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T135107578603Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23a703ba13b3e5d5c0ea621db8e633934eefaf803fe11167896a2235ec39166a +size 5918061 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T135108890669Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T135108890669Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9edf6d8d791014515758b23611533287daac333 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T135108890669Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:08.890267Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:07.685432Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T135108890669Z.pickle", + "timestamp_utc": "2026-08-01T13:51:08.986529Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T135108890669Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T135108890669Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bffd91a9a59b7f8f9d0ffda8777bc9c6c0a8f63d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T135108890669Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9580689450449f158c34fc59cccb63f781a84c7042bb63228db39ea69fc45812 +size 5918073 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T135110037371Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T135110037371Z.json new file mode 100644 index 0000000000000000000000000000000000000000..df763a77e2ec3846497310395360a43b5f67d72d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T135110037371Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:10.036951Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 6217829376, + "inactive_split_bytes_peak": 8354811904, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:08.997664Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 8354811904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T135110037371Z.pickle", + "timestamp_utc": "2026-08-01T13:51:10.141184Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T135110037371Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T135110037371Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f20ab8896c501f7d4c4c6831a1709812646823a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T135110037371Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f4d2d90df8fd3255c13b552a8cae22815b1a22ad91da29983562cd30bb7e266 +size 5918057 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T135205052129Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T135205052129Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dc259dd5f90d92ebcce845fff8e04685203aca46 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T135205052129Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105787282944, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 8858143744, + "inactive_split_bytes_peak": 9172658688, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:52:05.051655Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105787282944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 105787282944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787282944, + "device": 6, + "inactive_split_bytes_current": 6687591424, + "inactive_split_bytes_peak": 8354811904, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 107562926080, + "reserved_bytes_peak": 107562926080, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:10.153483Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8858143744, + "inactive_split_bytes_current_delta": 2170552320, + "inactive_split_bytes_peak": 9172658688, + "inactive_split_bytes_peak_delta": 817846784, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 107562926080, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107562926080, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T135205052129Z.pickle", + "timestamp_utc": "2026-08-01T13:52:05.254741Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T135205052129Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T135205052129Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2be8ab69fa070bbb264c8ee96e28967ee9fe1f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T135205052129Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b199bff8c9e8a15b2477a2c7fd630b0d49d10a6c27443f22c4fcebad811efabc +size 9679139 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T135100155265Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T135100155265Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31a3116e0633c449b7a6ca15553d84531a388114 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T135100155265Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 37208595968, + "active_bytes_peak": 82152062976, + "active_bytes_peak_delta": 46685726720, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 82152062976, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 82152062976, + "device": 7, + "inactive_split_bytes_current": 7392233984, + "inactive_split_bytes_peak": 7456688128, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 94736744448, + "reserved_bytes_peak": 94736744448, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T13:51:00.154657Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 37208595968, + "allocated_bytes_peak": 82152062976, + "allocated_bytes_peak_delta": 46685726720, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:49:52.072336Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7392233984, + "inactive_split_bytes_current_delta": 7259415040, + "inactive_split_bytes_peak": 7456688128, + "inactive_split_bytes_peak_delta": 7296302080, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 94736744448, + "reserved_bytes_current_delta": 55492739072, + "reserved_bytes_peak": 94736744448, + "reserved_bytes_peak_delta": 55492739072, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T135100155265Z.pickle", + "timestamp_utc": "2026-08-01T13:51:00.272466Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T135100155265Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T135100155265Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..98ae8c79b80fa7d8638c29e4b814e66f1709bbab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T135100155265Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa1382e1cb5f7e714f87712c97f378567e97106462f0155c401a510b05eedad6 +size 5925467 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T135101378533Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T135101378533Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f66e03c29d0266c33906ad0aef4d7562cbbf1a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T135101378533Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105780979712, + "active_bytes_peak_delta": 23628916736, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105780979712, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105780979712, + "device": 7, + "inactive_split_bytes_current": 6450612736, + "inactive_split_bytes_peak": 7638075392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107084775424, + "reserved_bytes_peak": 107084775424, + "segment_count_current": 479, + "segment_count_peak": 479, + "timestamp_utc": "2026-08-01T13:51:01.378069Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105780979712, + "allocated_bytes_peak_delta": 23628916736, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 82152062976, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 82152062976, + "device": 7, + "inactive_split_bytes_current": 7392233984, + "inactive_split_bytes_peak": 7456688128, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 94736744448, + "reserved_bytes_peak": 94736744448, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T13:51:00.291193Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6450612736, + "inactive_split_bytes_current_delta": -941621248, + "inactive_split_bytes_peak": 7638075392, + "inactive_split_bytes_peak_delta": 181387264, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107084775424, + "reserved_bytes_current_delta": 12348030976, + "reserved_bytes_peak": 107084775424, + "reserved_bytes_peak_delta": 12348030976, + "segment_count_current": 479, + "segment_count_current_delta": 59, + "segment_count_peak": 479, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T135101378533Z.pickle", + "timestamp_utc": "2026-08-01T13:51:01.500534Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T135101378533Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T135101378533Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fdc69913e1277a199dd6e81ed9370eabc14764c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T135101378533Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3437c16b2254d8ee907ef95fb8a74d777fd25c009f1bffc9f2090eb662234963 +size 5917239 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T135102524621Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T135102524621Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e36479e052bdf5f64e5d2ac3048cc2e738fec98 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T135102524621Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784387072, + "active_bytes_peak_delta": 3407360, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105784387072, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105784387072, + "device": 7, + "inactive_split_bytes_current": 6391892480, + "inactive_split_bytes_peak": 8284988416, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:02.524170Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784387072, + "allocated_bytes_peak_delta": 3407360, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105780979712, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105780979712, + "device": 7, + "inactive_split_bytes_current": 6450612736, + "inactive_split_bytes_peak": 7638075392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107084775424, + "reserved_bytes_peak": 107084775424, + "segment_count_current": 479, + "segment_count_peak": 479, + "timestamp_utc": "2026-08-01T13:51:01.523262Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6391892480, + "inactive_split_bytes_current_delta": -58720256, + "inactive_split_bytes_peak": 8284988416, + "inactive_split_bytes_peak_delta": 646913024, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107554537472, + "reserved_bytes_current_delta": 469762048, + "reserved_bytes_peak": 107554537472, + "reserved_bytes_peak_delta": 469762048, + "segment_count_current": 480, + "segment_count_current_delta": 1, + "segment_count_peak": 480, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T135102524621Z.pickle", + "timestamp_utc": "2026-08-01T13:51:02.637555Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T135102524621Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T135102524621Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a4714aee9822ca6163ed9d592bd11dcad809b0bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T135102524621Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43b88e2677cd65f8cf1796bd41de4f9c0fef83c685cafde9274d6f4019dc89d5 +size 5921330 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T135103906337Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T135103906337Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61ede4b7d92eabc8db4e77f09ad1eeb8712a6aa3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T135103906337Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105784780288, + "active_bytes_peak_delta": 393216, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105784780288, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105784780288, + "device": 7, + "inactive_split_bytes_current": 6626773504, + "inactive_split_bytes_peak": 8284988416, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:03.905873Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784780288, + "allocated_bytes_peak_delta": 393216, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105784387072, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105784387072, + "device": 7, + "inactive_split_bytes_current": 6391892480, + "inactive_split_bytes_peak": 8284988416, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:02.654425Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6626773504, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 8284988416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107554537472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107554537472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T135103906337Z.pickle", + "timestamp_utc": "2026-08-01T13:51:04.020644Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T135103906337Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T135103906337Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ac77925fa6fc2e2948aa36ad438722f4f17d15d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T135103906337Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60f4efc07224e6c5a2880059f2eb5b1945c29359117761a33146d97d98edfeff +size 5920105 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T135105298090Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T135105298090Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed6499e60a80fd45923b2ea7eb6c480e0b1092cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T135105298090Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105786132480, + "active_bytes_peak_delta": 1352192, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105786132480, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105786132480, + "device": 7, + "inactive_split_bytes_current": 6630967808, + "inactive_split_bytes_peak": 8342922240, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:05.297647Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105786132480, + "allocated_bytes_peak_delta": 1352192, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105784780288, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105784780288, + "device": 7, + "inactive_split_bytes_current": 6626773504, + "inactive_split_bytes_peak": 8284988416, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:04.036901Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6630967808, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8342922240, + "inactive_split_bytes_peak_delta": 57933824, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107554537472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107554537472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T135105298090Z.pickle", + "timestamp_utc": "2026-08-01T13:51:05.408337Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T135105298090Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T135105298090Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9875ef7bb058bb897812954a157bd9ad868ea5c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T135105298090Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9f129c52ec0ba558b54b33a10956e2440313f6f955b52141fb046c1eace05f0 +size 5919317 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T135106432588Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T135106432588Z.json new file mode 100644 index 0000000000000000000000000000000000000000..48bfab01f8a5008da30099fff1a44adae2ff20de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T135106432588Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105786877952, + "active_bytes_peak_delta": 745472, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105786877952, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105786877952, + "device": 7, + "inactive_split_bytes_current": 7096535552, + "inactive_split_bytes_peak": 8579900416, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:06.432193Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105786877952, + "allocated_bytes_peak_delta": 745472, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105786132480, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105786132480, + "device": 7, + "inactive_split_bytes_current": 6630967808, + "inactive_split_bytes_peak": 8342922240, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:05.424593Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7096535552, + "inactive_split_bytes_current_delta": 465567744, + "inactive_split_bytes_peak": 8579900416, + "inactive_split_bytes_peak_delta": 236978176, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107554537472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107554537472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T135106432588Z.pickle", + "timestamp_utc": "2026-08-01T13:51:06.542045Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T135106432588Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T135106432588Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4efb768c42d63cd7ba51abb5d1a418b0338c78ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T135106432588Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19dd44612cbba956e20ab8cf51843f298926d5af71cf61a35214f6f85748a1bc +size 5919348 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T135107578645Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T135107578645Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7ce9edea9e3b1c62da4d71b62db12dbdb0b0abbe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T135107578645Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787553792, + "active_bytes_peak_delta": 675840, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105787553792, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105787553792, + "device": 7, + "inactive_split_bytes_current": 6391892480, + "inactive_split_bytes_peak": 8579900416, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:07.578242Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787553792, + "allocated_bytes_peak_delta": 675840, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105786877952, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105786877952, + "device": 7, + "inactive_split_bytes_current": 7096535552, + "inactive_split_bytes_peak": 8579900416, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:06.558091Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6391892480, + "inactive_split_bytes_current_delta": -704643072, + "inactive_split_bytes_peak": 8579900416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107554537472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107554537472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T135107578645Z.pickle", + "timestamp_utc": "2026-08-01T13:51:07.687324Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T135107578645Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T135107578645Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3fcdc08e3534152b005780e667f59b2cbf19f31 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T135107578645Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16d9f214c09569c1b7d6e1924a133f9f3ae463c6f25e30c922d0682b765f00df +size 5918772 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T135108890731Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T135108890731Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6bec346b6cdd4904bf8bfdd55d8ea4d7dc6c6c60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T135108890731Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787553792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105787553792, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105787553792, + "device": 7, + "inactive_split_bytes_current": 6274451968, + "inactive_split_bytes_peak": 8579900416, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:08.890291Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787553792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105787553792, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105787553792, + "device": 7, + "inactive_split_bytes_current": 6391892480, + "inactive_split_bytes_peak": 8579900416, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:07.703146Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6274451968, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 8579900416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107554537472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107554537472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T135108890731Z.pickle", + "timestamp_utc": "2026-08-01T13:51:08.998852Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T135108890731Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T135108890731Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e54e3ba57ce1ec879ecb27f516cd471fbcce7bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T135108890731Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12dae528ec32a3fafa952ada9af40b78d6362ef6fa60c6774de5f90bbdfcd167 +size 5918756 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T135110037456Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T135110037456Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7dd14c548b24e64315f61520299d865025d16898 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T135110037456Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105787553792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105787553792, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105787553792, + "device": 7, + "inactive_split_bytes_current": 7096535552, + "inactive_split_bytes_peak": 8579900416, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:10.037004Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787553792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105787553792, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105787553792, + "device": 7, + "inactive_split_bytes_current": 6274451968, + "inactive_split_bytes_peak": 8579900416, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:09.015110Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7096535552, + "inactive_split_bytes_current_delta": 822083584, + "inactive_split_bytes_peak": 8579900416, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107554537472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107554537472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T135110037456Z.pickle", + "timestamp_utc": "2026-08-01T13:51:10.147719Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T135110037456Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T135110037456Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..63d58de28a6d0595293b4c83a4917c85d661f5bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T135110037456Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05b4aa477cf997cd58290e4de0013f04d0f4692b6b8cab8d0c2369a0110b9eb7 +size 5918847 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T135205052121Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T135205052121Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1735c5fd803bf3c58c7f45a82ea3f4b7593d507c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T135205052121Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165762048, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 105787553792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165762048, + "active_bytes_peak": 105787553792, + "allocated_bytes_current": 61165762048, + "allocated_bytes_peak": 105787553792, + "device": 7, + "inactive_split_bytes_current": 8623262208, + "inactive_split_bytes_peak": 8904040960, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:52:05.051669Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165762048, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105787553792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 105787553792, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105787553792, + "device": 7, + "inactive_split_bytes_current": 7096535552, + "inactive_split_bytes_peak": 8579900416, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 107554537472, + "reserved_bytes_peak": 107554537472, + "segment_count_current": 480, + "segment_count_peak": 480, + "timestamp_utc": "2026-08-01T13:51:10.164635Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8623262208, + "inactive_split_bytes_current_delta": 1526726656, + "inactive_split_bytes_peak": 8904040960, + "inactive_split_bytes_peak_delta": 324140544, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 107554537472, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107554537472, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 480, + "segment_count_current_delta": 0, + "segment_count_peak": 480, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T135205052121Z.pickle", + "timestamp_utc": "2026-08-01T13:52:05.263791Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T135205052121Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T135205052121Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5376cc8dd38ebc05e1f4a817fba63eb8da47a4e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T135205052121Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb4e3acd96b26ff353a90bbdeb0c8f56dd5977e99efc82f3d4da93d0fe2fbd03 +size 9688747 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..13b0b232c267559fb9ee2b51a1e5754159f46465 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 2048 +capacity: 1.50 +expert_dispatch_overlap: on +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6034 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 220 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T135449863545Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T135449863545Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4e4dafb0566cf52613cda3db4933a721ebe3413 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T135449863545Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 31622347776, + "active_bytes_peak": 82173430784, + "active_bytes_peak_delta": 46707094528, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 82173430784, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 82173430784, + "device": 0, + "inactive_split_bytes_current": 7293103104, + "inactive_split_bytes_peak": 7293103616, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98748596224, + "reserved_bytes_peak": 98748596224, + "segment_count_current": 504, + "segment_count_peak": 504, + "timestamp_utc": "2026-08-01T13:54:49.863107Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 31622347776, + "allocated_bytes_peak": 82173430784, + "allocated_bytes_peak_delta": 46707094528, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:53:43.246085Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7293103104, + "inactive_split_bytes_current_delta": 7158187008, + "inactive_split_bytes_peak": 7293103616, + "inactive_split_bytes_peak_delta": 7132471808, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98748596224, + "reserved_bytes_current_delta": 59502493696, + "reserved_bytes_peak": 98748596224, + "reserved_bytes_peak_delta": 59502493696, + "segment_count_current": 504, + "segment_count_current_delta": 432, + "segment_count_peak": 504, + "segment_count_peak_delta": 432, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T135449863545Z.pickle", + "timestamp_utc": "2026-08-01T13:54:49.974601Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T135449863545Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T135449863545Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..518979d3166aa38eae7e3036ce1f1c7430b9e408 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T135449863545Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:855db8f1c83f645c9e5001f29cb73aa603a8952834f91ec8bc1a3bdf0609b5b4 +size 5395189 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T135450901837Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T135450901837Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e6dfd62b911bf2c9eb03c4f539b762ec540b490 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T135450901837Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105812466688, + "active_bytes_peak_delta": 23639035904, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105812466688, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105812466688, + "device": 0, + "inactive_split_bytes_current": 7307783168, + "inactive_split_bytes_peak": 8487830528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107808292864, + "reserved_bytes_peak": 107808292864, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:50.901428Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105812466688, + "allocated_bytes_peak_delta": 23639035904, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 82173430784, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 82173430784, + "device": 0, + "inactive_split_bytes_current": 7293103104, + "inactive_split_bytes_peak": 7293103616, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98748596224, + "reserved_bytes_peak": 98748596224, + "segment_count_current": 504, + "segment_count_peak": 504, + "timestamp_utc": "2026-08-01T13:54:49.996016Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7307783168, + "inactive_split_bytes_current_delta": 14680064, + "inactive_split_bytes_peak": 8487830528, + "inactive_split_bytes_peak_delta": 1194726912, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107808292864, + "reserved_bytes_current_delta": 9059696640, + "reserved_bytes_peak": 107808292864, + "reserved_bytes_peak_delta": 9059696640, + "segment_count_current": 538, + "segment_count_current_delta": 34, + "segment_count_peak": 538, + "segment_count_peak_delta": 34, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T135450901837Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.013343Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T135450901837Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T135450901837Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b848fea9bcc78be54c0dfcb5d83f2f1fba606a37 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T135450901837Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7869fbcbfb2c27ac1dfc2efc47faa355e65caca7877a699d3aeaf3b79889ce00 +size 5620721 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T135451887014Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T135451887014Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9bd9981678d3131641828872818eb548cd1035d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T135451887014Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105812466688, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105812466688, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105812466688, + "device": 0, + "inactive_split_bytes_current": 7305686016, + "inactive_split_bytes_peak": 8487830528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 107808292864, + "reserved_bytes_peak": 107808292864, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:51.886610Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105812466688, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105812466688, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105812466688, + "device": 0, + "inactive_split_bytes_current": 7307783168, + "inactive_split_bytes_peak": 8487830528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107808292864, + "reserved_bytes_peak": 107808292864, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:51.029885Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7305686016, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8487830528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 107808292864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107808292864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T135451887014Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.992698Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T135451887014Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T135451887014Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c20db7cf151f533d6859d8fb4b8470ee7175e12 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T135451887014Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb1f7d3b4993d4a8f15438605baedd8cee54826ba0aeddd7b9a9996fe6cf8d65 +size 5613113 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T135452888507Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T135452888507Z.json new file mode 100644 index 0000000000000000000000000000000000000000..415c06c33789b2c1839d998fc082ff7d28981341 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T135452888507Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813116928, + "active_bytes_peak_delta": 650240, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 7070804992, + "inactive_split_bytes_peak": 8487830528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:54:52.888088Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813116928, + "allocated_bytes_peak_delta": 650240, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105812466688, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105812466688, + "device": 0, + "inactive_split_bytes_current": 7305686016, + "inactive_split_bytes_peak": 8487830528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 107808292864, + "reserved_bytes_peak": 107808292864, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:52.008701Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7070804992, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 8487830528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_current_delta": 882900992, + "reserved_bytes_peak": 108691193856, + "reserved_bytes_peak_delta": 882900992, + "segment_count_current": 540, + "segment_count_current_delta": 2, + "segment_count_peak": 540, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T135452888507Z.pickle", + "timestamp_utc": "2026-08-01T13:54:52.992662Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T135452888507Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T135452888507Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf5430178e01d616f4854328fb8bff5ffa4a0c7b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T135452888507Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aef7990197cbd76616cd55d2161fe7ff86b654610d00522a8f1ba1d3c17d58bd +size 5617620 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T135454422328Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T135454422328Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3bd6cfafcb04bb448d7854a9a87e164ebddffd97 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T135454422328Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813116928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 7064513536, + "inactive_split_bytes_peak": 8487830528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:54:54.421887Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813116928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 7070804992, + "inactive_split_bytes_peak": 8487830528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:54:53.008586Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7064513536, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 8487830528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108691193856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 540, + "segment_count_current_delta": 0, + "segment_count_peak": 540, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T135454422328Z.pickle", + "timestamp_utc": "2026-08-01T13:54:54.533310Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T135454422328Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T135454422328Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ad670723b65aed36fe286f8c8d4a053df1d07bf5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T135454422328Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fb33d156854b6a43c19e2383056df8753a27131e87da815a7ab78c09b168685 +size 5617057 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T135455442920Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T135455442920Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c46eabd185b9be7978a81dcc559081c84820992e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T135455442920Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813116928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 7062416384, + "inactive_split_bytes_peak": 8487830528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:54:55.442514Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813116928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 7064513536, + "inactive_split_bytes_peak": 8487830528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:54:54.551269Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7062416384, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8487830528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108691193856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 540, + "segment_count_current_delta": 0, + "segment_count_peak": 540, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T135455442920Z.pickle", + "timestamp_utc": "2026-08-01T13:54:55.545460Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T135455442920Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T135455442920Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea4d44d8a848ae92e93808d9b0e42f9ba2ea25df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T135455442920Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a79efb3c6d2e142f779c0600146f7f155418b8fc85b22f1fdaa30ab1bd46c32 +size 5617882 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T135456452360Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T135456452360Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5b690418b1ec35238be1ce8a699a569ff572ef7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T135456452360Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813116928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 7066610688, + "inactive_split_bytes_peak": 8487830528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:54:56.451948Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813116928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 7062416384, + "inactive_split_bytes_peak": 8487830528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:54:55.561069Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7066610688, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8487830528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108691193856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 540, + "segment_count_current_delta": 0, + "segment_count_peak": 540, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T135456452360Z.pickle", + "timestamp_utc": "2026-08-01T13:54:56.555042Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T135456452360Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T135456452360Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9461def366c0addba1e95133f1ccac481ddcbf11 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T135456452360Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b0477ee357d5892aec74f3556c08b24c30b8a566612351b660f11bf62644e5d +size 5615441 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T135457451756Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T135457451756Z.json new file mode 100644 index 0000000000000000000000000000000000000000..750ea398e221210638a56580cc22df19670af05b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T135457451756Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813116928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 7064513536, + "inactive_split_bytes_peak": 8487830528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:54:57.451352Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813116928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 7066610688, + "inactive_split_bytes_peak": 8487830528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:54:56.570734Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7064513536, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8487830528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108691193856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 540, + "segment_count_current_delta": 0, + "segment_count_peak": 540, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T135457451756Z.pickle", + "timestamp_utc": "2026-08-01T13:54:57.552929Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T135457451756Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T135457451756Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3cdd86b54404e95eb15a35df0b28320f50b0b69f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T135457451756Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05796de4a4e66dd16a8ebdef8c203fe9fb2febf33dc3b2fa39312b236eeb7b83 +size 5617113 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T135458453941Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T135458453941Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e1ffe51b27c3067244359a9be146ca055512f8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T135458453941Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813116928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 7068707840, + "inactive_split_bytes_peak": 8487830528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:54:58.453540Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813116928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 7064513536, + "inactive_split_bytes_peak": 8487830528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:54:57.568424Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7068707840, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8487830528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108691193856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 540, + "segment_count_current_delta": 0, + "segment_count_peak": 540, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T135458453941Z.pickle", + "timestamp_utc": "2026-08-01T13:54:58.554813Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T135458453941Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T135458453941Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..127610f5528b81ad0ab67e497d866500f2fb9cf0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T135458453941Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e0f1635d3bd607ff2800064bc7dc2f2758887ae9e866049ebe363328a4f83d7 +size 5617314 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T135551183481Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T135551183481Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bd33a66f9c0c57e5c66470bc22dafb4f45785e6a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T135551183481Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803160064, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105813116928, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803160064, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66803160064, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 6794292224, + "inactive_split_bytes_peak": 8487830528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:55:51.183007Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803160064, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105813116928, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 105813116928, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 105813116928, + "device": 0, + "inactive_split_bytes_current": 7068707840, + "inactive_split_bytes_peak": 8487830528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_peak": 108691193856, + "segment_count_current": 540, + "segment_count_peak": 540, + "timestamp_utc": "2026-08-01T13:54:58.570392Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6794292224, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 8487830528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 108691193856, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108691193856, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 540, + "segment_count_current_delta": 0, + "segment_count_peak": 540, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T135551183481Z.pickle", + "timestamp_utc": "2026-08-01T13:55:51.414347Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T135551183481Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T135551183481Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..79ef2463b7b72cc4862866d7989ed94bed19be34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T135551183481Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a9dfa2a4f925c5783af625b483a6f2d0207d6e17be5736a56a28d05325a66bc +size 10191516 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T135449863481Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T135449863481Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96bdc265c6ca8bd900cf283d1568a88dd71bd932 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T135449863481Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 82172224512, + "active_bytes_peak_delta": 46705888256, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 82172224512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 82172224512, + "device": 1, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_peak": 7289163264, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 98746499072, + "reserved_bytes_peak": 98746499072, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:54:49.862884Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 82172224512, + "allocated_bytes_peak_delta": 46705888256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:53:43.246272Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_current_delta": 7156343808, + "inactive_split_bytes_peak": 7289163264, + "inactive_split_bytes_peak_delta": 7128777216, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 98746499072, + "reserved_bytes_current_delta": 59502493696, + "reserved_bytes_peak": 98746499072, + "reserved_bytes_peak_delta": 59502493696, + "segment_count_current": 503, + "segment_count_current_delta": 432, + "segment_count_peak": 503, + "segment_count_peak_delta": 432, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T135449863481Z.pickle", + "timestamp_utc": "2026-08-01T13:54:49.974829Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T135449863481Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T135449863481Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..91879b7ed7171116feb0c5c6dea4c282cf493a47 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T135449863481Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1350d9ebbe0e62014de745e5c833eb834a58e8561d4731a810c1548d01f860bd +size 5385161 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T135450901782Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T135450901782Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ee2679f7ca32ddbfc16d916312e5348dc0ab4cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T135450901782Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 23643720192, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_peak": 7423191040, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:50.901372Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 23643720192, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 82172224512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 82172224512, + "device": 1, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_peak": 7289163264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 98746499072, + "reserved_bytes_peak": 98746499072, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:54:49.993059Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 7423191040, + "inactive_split_bytes_peak_delta": 134027776, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108055756800, + "reserved_bytes_current_delta": 9309257728, + "reserved_bytes_peak": 108055756800, + "reserved_bytes_peak_delta": 9309257728, + "segment_count_current": 535, + "segment_count_current_delta": 32, + "segment_count_peak": 535, + "segment_count_peak_delta": 32, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T135450901782Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.011655Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T135450901782Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T135450901782Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..68d4b5cdf7441b88d2ed623ea51668fa6d018d76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T135450901782Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee61353b1f239ac8aeef4b4faab064cabfb6860cd48c77b2b72afd55e19e836d +size 5617574 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T135451887069Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T135451887069Z.json new file mode 100644 index 0000000000000000000000000000000000000000..291010d2d9eea30ee218016b4fc1cd743cf02bf2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T135451887069Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 7914496000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:51.886629Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_peak": 7423191040, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:51.025042Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7914496000, + "inactive_split_bytes_peak_delta": 491304960, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108055756800, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108055756800, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T135451887069Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.991524Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T135451887069Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T135451887069Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1f5bc19039d34320c2f4acd8609ea5ddd7b66b24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T135451887069Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d20f9f852f8e2298e7af0a6ed5988687b98215da7417b9c6c8048905f76ebb70 +size 5610030 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T135452888355Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T135452888355Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9076577ccd9ef70cc1bcd51c5d6d7e00ffd6012 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T135452888355Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 7914496000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:52.887937Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 7914496000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:52.004829Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 7914496000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108055756800, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108055756800, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T135452888355Z.pickle", + "timestamp_utc": "2026-08-01T13:54:52.989207Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T135452888355Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T135452888355Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..af3ca501a842cbd0c0b89bbfe15da60054bb454a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T135452888355Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57b6431a3869b1251a3d7c4914eea59abecda391466d1feb9a3ea9f47724010f +size 5611121 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T135454422204Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T135454422204Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e59d9ada2a30595c5fb54d88be854935a66581e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T135454422204Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 7914496000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:54.421787Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 7914496000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:53.002357Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 7914496000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108055756800, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108055756800, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T135454422204Z.pickle", + "timestamp_utc": "2026-08-01T13:54:54.531843Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T135454422204Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T135454422204Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..830e2911e92558a07326280f8ae7087a393e2106 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T135454422204Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6ed020fbe140c8b0d408e18d4d6f61926378bf83d3acd4add481a23db0c08c4 +size 5609615 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T135455442843Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T135455442843Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5bf033a0109e26aa154e2e70b7ea8d72c9fdffd9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T135455442843Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7060573184, + "inactive_split_bytes_peak": 7914496000, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:55.442439Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 7914496000, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:54.547350Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7060573184, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7914496000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108055756800, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108055756800, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T135455442843Z.pickle", + "timestamp_utc": "2026-08-01T13:54:55.543107Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T135455442843Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T135455442843Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0b4ba9e2768ea4a9437a98407f7df40785406633 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T135455442843Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c516f401a341addc307013fbfd066897a9d3ef6289f90af0b148b56e84d391af +size 5615651 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T135456452252Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T135456452252Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b552c928409261d264168c3745d3825bd95d377 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T135456452252Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7064767488, + "inactive_split_bytes_peak": 7914496000, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:56.451849Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7060573184, + "inactive_split_bytes_peak": 7914496000, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:55.556126Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7064767488, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 7914496000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108055756800, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108055756800, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T135456452252Z.pickle", + "timestamp_utc": "2026-08-01T13:54:56.551766Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T135456452252Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T135456452252Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e6ecc410ad521e0fc4dc05823d2a7ed3bc5ab20 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T135456452252Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee4de5d9346fe13c532359f55110a96e9e8789230df49d81bbb1bb546ea7895e +size 5609607 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T135457451671Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T135457451671Z.json new file mode 100644 index 0000000000000000000000000000000000000000..078e211e5b4783739564fcde80bc25d8e5b10eec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T135457451671Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 7914496000, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:57.451267Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7064767488, + "inactive_split_bytes_peak": 7914496000, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:56.564713Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7914496000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108055756800, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108055756800, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T135457451671Z.pickle", + "timestamp_utc": "2026-08-01T13:54:57.549861Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T135457451671Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T135457451671Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..875a602bab66c1ff05c8443e2d471791e34f19f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T135457451671Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c10bf5e028cd0d4ff96fbebe7d9fb49da59581d3f383dbbf1c338e55a4c5b081 +size 5615554 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T135458453830Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T135458453830Z.json new file mode 100644 index 0000000000000000000000000000000000000000..675c65c7f661d2535ec633558854293c06beecd1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T135458453830Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7301745664, + "inactive_split_bytes_peak": 7922830336, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:58.453433Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 7914496000, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:57.562483Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7301745664, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 7922830336, + "inactive_split_bytes_peak_delta": 8334336, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108055756800, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108055756800, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T135458453830Z.pickle", + "timestamp_utc": "2026-08-01T13:54:58.551313Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T135458453830Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T135458453830Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ef37a2585e9e514d7420fffc89155420ed8fc7e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T135458453830Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a97c2630abbfce2373df02f86aed81cc692a087020fd18754c6f719f7b53b7bd +size 5609744 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T135551183416Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T135551183416Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ced29c6d6650c4409766142819f8bf2dc81dcbf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T135551183416Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 6792449024, + "inactive_split_bytes_peak": 8037125120, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:55:51.182930Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 1, + "inactive_split_bytes_current": 7301745664, + "inactive_split_bytes_peak": 7922830336, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 108055756800, + "reserved_bytes_peak": 108055756800, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:58.564067Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6792449024, + "inactive_split_bytes_current_delta": -509296640, + "inactive_split_bytes_peak": 8037125120, + "inactive_split_bytes_peak_delta": 114294784, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 108055756800, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108055756800, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T135551183416Z.pickle", + "timestamp_utc": "2026-08-01T13:55:51.385465Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T135551183416Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T135551183416Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..90135a9c9d016ad4bcf1b96022f0b0521057eb4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T135551183416Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:542fdb1d5b92d03d635ebbbcf8eeff1230c37dbc1cf8add9ffef0f4810d0a666 +size 10204075 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T135449863350Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T135449863350Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2611cb33dd18d5402a2a7e8dfc6df192e03c6de0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T135449863350Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 82172224512, + "active_bytes_peak_delta": 46705888256, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 82172224512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 82172224512, + "device": 2, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_peak": 7289163264, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 99216261120, + "reserved_bytes_peak": 99216261120, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:54:49.862749Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 82172224512, + "allocated_bytes_peak_delta": 46705888256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:53:43.246260Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_current_delta": 7156343808, + "inactive_split_bytes_peak": 7289163264, + "inactive_split_bytes_peak_delta": 7128777216, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 99216261120, + "reserved_bytes_current_delta": 59972255744, + "reserved_bytes_peak": 99216261120, + "reserved_bytes_peak_delta": 59972255744, + "segment_count_current": 505, + "segment_count_current_delta": 434, + "segment_count_peak": 505, + "segment_count_peak_delta": 434, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T135449863350Z.pickle", + "timestamp_utc": "2026-08-01T13:54:49.977792Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T135449863350Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T135449863350Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6b80ca7ca64fc791edc04fb23d6b70666fd8ada --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T135449863350Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd620f7a40d9bdf1a1ea70a7834e611c62acb760b0eadb82909c329258115294 +size 5385505 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T135450901780Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T135450901780Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ca6e3ad409c440dd0ce3f7c69ebf3627479107e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T135450901780Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814371840, + "active_bytes_peak_delta": 23642147328, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_peak": 8245088256, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:50.901364Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814371840, + "allocated_bytes_peak_delta": 23642147328, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 82172224512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 82172224512, + "device": 2, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_peak": 7289163264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 99216261120, + "reserved_bytes_peak": 99216261120, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:54:49.998959Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 8245088256, + "inactive_split_bytes_peak_delta": 955924992, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 9779019776, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 9779019776, + "segment_count_current": 538, + "segment_count_current_delta": 33, + "segment_count_peak": 538, + "segment_count_peak_delta": 33, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T135450901780Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.014318Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T135450901780Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T135450901780Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44b0432c22494b6fa62e03aab0a8a69fa59273d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T135450901780Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5982226a52d45461aa739b4ff3d57211bed6f24eedd65025c94ed8f232e5053c +size 5617749 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T135451886912Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T135451886912Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eea138e1a8272bdaabc4ba81700973b7bd385203 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T135451886912Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814371840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 8245088256, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:51.886520Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814371840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_peak": 8245088256, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:51.030313Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8245088256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T135451886912Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.994602Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T135451886912Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T135451886912Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..28bfd94b9751f42594f5d0a25a7dabc3f11b398e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T135451886912Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfb0233d61887b8f7cac8f337980ee0ef254fd4479ad3f0d167e23259290f90d +size 5610802 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T135452888343Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T135452888343Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5473186fdb695bad69887186f95c7da264a233d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T135452888343Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814371840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 8245088256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:52.887932Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814371840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 8245088256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:52.010657Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8245088256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T135452888343Z.pickle", + "timestamp_utc": "2026-08-01T13:54:52.993886Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T135452888343Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T135452888343Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..723143c8c39faa8a46276a36fe1bd465c4b18d47 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T135452888343Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc60a0e18e3ce1e7cc9546223d16ce8acd83ea78a841325bf54ef3572b624bbb +size 5611891 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T135454422199Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T135454422199Z.json new file mode 100644 index 0000000000000000000000000000000000000000..22de02af8204ac71cd8c071950094e847bce0260 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T135454422199Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814371840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8245088256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:54.421778Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814371840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 8245088256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:53.009409Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 8245088256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T135454422199Z.pickle", + "timestamp_utc": "2026-08-01T13:54:54.528656Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T135454422199Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T135454422199Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..518f24501b69f6594e905bc958b290d197de25ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T135454422199Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13f68e571b5f6d3787689c66b6df0477259aebf6dc07184d4b1f3c574f1c9f5a +size 5610382 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T135455442859Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T135455442859Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f215efd83310fd1f8f056bf2dbb71ca24f01ed44 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T135455442859Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814371840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7060573184, + "inactive_split_bytes_peak": 8245088256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:55.442454Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814371840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8245088256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:54.544311Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7060573184, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8245088256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T135455442859Z.pickle", + "timestamp_utc": "2026-08-01T13:54:55.547890Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T135455442859Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T135455442859Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..12d5803dbe5c5472c9e84c3a9f08e4eefe450976 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T135455442859Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01992f4d098bbd1fe146b055204954feea0c982b03d262e17e01f029d5b51897 +size 5616425 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T135456452314Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T135456452314Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e8782bd37709dfb3a61fab65b0100b9cb7efe87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T135456452314Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814371840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7064767488, + "inactive_split_bytes_peak": 8245088256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:56.451885Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814371840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7060573184, + "inactive_split_bytes_peak": 8245088256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:55.563432Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7064767488, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8245088256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T135456452314Z.pickle", + "timestamp_utc": "2026-08-01T13:54:56.556301Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T135456452314Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T135456452314Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e41fa7fd7a47ac29d92d8a95097ce921c9a2a3d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T135456452314Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a19c69d64b4f0756802ca212ad6326796b300565430ca6d093a3298cdfed8540 +size 5610361 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T135457451677Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T135457451677Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9da151f8845a6dc0e340062471d7d3f2b763eda2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T135457451677Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814371840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8245088256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:57.451264Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814371840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7064767488, + "inactive_split_bytes_peak": 8245088256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:56.571851Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8245088256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T135457451677Z.pickle", + "timestamp_utc": "2026-08-01T13:54:57.554441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T135457451677Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T135457451677Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d821c603a2b439953bc0b073887f8fd59ea32908 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T135457451677Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e49176dfdcd801f14924b2577372ebce98ed2f37ad3251ac4a15bcb49228c334 +size 5616394 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T135458453838Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T135458453838Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d2eb92198743db8f203b25e0e1c493eb2226ddc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T135458453838Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814371840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7066864640, + "inactive_split_bytes_peak": 8245088256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:58.453441Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814371840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8245088256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:57.569957Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7066864640, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8245088256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T135458453838Z.pickle", + "timestamp_utc": "2026-08-01T13:54:58.555854Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T135458453838Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T135458453838Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..085abd6809f74197dbcb831ecd9e05873c7cbf70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T135458453838Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61f3505e998076bd6fdb45ac771973e66c3640fdb844f7ae938617f772017e81 +size 5610514 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T135551183450Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T135551183450Z.json new file mode 100644 index 0000000000000000000000000000000000000000..54b4358ab0cd06f1304e7f8cf385ef86d908fca6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T135551183450Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105814371840, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 6792449024, + "inactive_split_bytes_peak": 8245088256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:55:51.182962Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105814371840, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105814371840, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105814371840, + "device": 2, + "inactive_split_bytes_current": 7066864640, + "inactive_split_bytes_peak": 8245088256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:58.571255Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6792449024, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 8245088256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T135551183450Z.pickle", + "timestamp_utc": "2026-08-01T13:55:51.392553Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T135551183450Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T135551183450Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7b01067a8ff5ff0231c2ec0293a980c175f807c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T135551183450Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc5a3e768c1804e5827b5e8ade8cdde0983b24c55114cf99c4f00cafe6faeb64 +size 10204982 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T135449863556Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T135449863556Z.json new file mode 100644 index 0000000000000000000000000000000000000000..46c79dec57739a159b36b987c2f63f90ffc8e4f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T135449863556Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 82172224512, + "active_bytes_peak_delta": 46705888256, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 82172224512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 82172224512, + "device": 3, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_peak": 7289163264, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 99216261120, + "reserved_bytes_peak": 99216261120, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:54:49.862943Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 82172224512, + "allocated_bytes_peak_delta": 46705888256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:53:43.246286Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_current_delta": 7156343808, + "inactive_split_bytes_peak": 7289163264, + "inactive_split_bytes_peak_delta": 7128777216, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 99216261120, + "reserved_bytes_current_delta": 59972255744, + "reserved_bytes_peak": 99216261120, + "reserved_bytes_peak_delta": 59972255744, + "segment_count_current": 505, + "segment_count_current_delta": 434, + "segment_count_peak": 505, + "segment_count_peak_delta": 434, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T135449863556Z.pickle", + "timestamp_utc": "2026-08-01T13:54:49.981030Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T135449863556Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T135449863556Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d7514e57d2adc4948de84a058adfefca91f013b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T135449863556Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dab3fb7a157d103977cbfdbc140ff43c4b10d1d51db2aa32a5b4bd3e37bf560d +size 5385516 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T135450901789Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T135450901789Z.json new file mode 100644 index 0000000000000000000000000000000000000000..584d4834bf614d55a2026b0c96ec7f5d8ca52b37 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T135450901789Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813847552, + "active_bytes_peak_delta": 23641623040, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_peak": 7649628160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 107760058368, + "reserved_bytes_peak": 107760058368, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:50.901377Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813847552, + "allocated_bytes_peak_delta": 23641623040, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 82172224512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 82172224512, + "device": 3, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_peak": 7289163264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 99216261120, + "reserved_bytes_peak": 99216261120, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:54:50.002059Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 7649628160, + "inactive_split_bytes_peak_delta": 360464896, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 107760058368, + "reserved_bytes_current_delta": 8543797248, + "reserved_bytes_peak": 107760058368, + "reserved_bytes_peak_delta": 8543797248, + "segment_count_current": 535, + "segment_count_current_delta": 30, + "segment_count_peak": 535, + "segment_count_peak_delta": 30, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T135450901789Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.013085Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T135450901789Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T135450901789Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..16ebaf6c79d07499123bec5280cec4ce73d1c95c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T135450901789Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d262b5e9673076909ec37170761b1aaf250306fe2148619608bc05f2bda39f4 +size 5617782 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T135451886937Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T135451886937Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6a73fbe26c5093c358a007b7763d91f58985eb16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T135451886937Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813847552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 8324517888, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:51.886542Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813847552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_peak": 7649628160, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 107760058368, + "reserved_bytes_peak": 107760058368, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T13:54:51.029488Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8324517888, + "inactive_split_bytes_peak_delta": 674889728, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 1235222528, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 1235222528, + "segment_count_current": 538, + "segment_count_current_delta": 3, + "segment_count_peak": 538, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T135451886937Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.994134Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T135451886937Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T135451886937Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..91e176664e7ec816c70cf9dcb90170f565c1a07d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T135451886937Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6fdb2c1e2de9caffb07305b920e90089ab1b792009ccbea8649dbde06470c7d +size 5597233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T135452888347Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T135452888347Z.json new file mode 100644 index 0000000000000000000000000000000000000000..33474fdf461758a6ef68f370dcb41f1656d3393d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T135452888347Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813847552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 8324517888, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:52.887938Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813847552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 8324517888, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:52.010156Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8324517888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T135452888347Z.pickle", + "timestamp_utc": "2026-08-01T13:54:52.994576Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T135452888347Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T135452888347Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..63a65962e8b80aeba2b857aa4f7b013f26bd39c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T135452888347Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:200a4673c14b7839323dbd8c298b3f01a74d8b91ba0b6a58cb40d0044e467a6e +size 5611826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T135454422337Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T135454422337Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c9c40873c04336ba9045e61fba53e9292aaae2ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T135454422337Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813847552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8324517888, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:54.421844Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813847552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 8324517888, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:53.010717Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 8324517888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T135454422337Z.pickle", + "timestamp_utc": "2026-08-01T13:54:54.530492Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T135454422337Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T135454422337Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..143db5021298fe3d5f556b7db421945d0c9bb075 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T135454422337Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bf9ed28cacd2d6b7afabc6df2e2fc7918a1bc508d53131cd5d998974be74969 +size 5610248 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T135455442895Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T135455442895Z.json new file mode 100644 index 0000000000000000000000000000000000000000..edf0c92d1dcda5296efab69262ecd16f39c0e726 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T135455442895Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813847552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7060573184, + "inactive_split_bytes_peak": 8324517888, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:55.442479Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813847552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8324517888, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:54.546650Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7060573184, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8324517888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T135455442895Z.pickle", + "timestamp_utc": "2026-08-01T13:54:55.549364Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T135455442895Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T135455442895Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3390c879fe7da751aa6cc66a72190d795b0a7631 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T135455442895Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba67a431f63535d758eddd216ff3fd1139374243e8e846b646d107e0cf44e083 +size 5615802 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T135456452301Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T135456452301Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e118ecae3f574834ccac617348a9ebb4aafc3ec2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T135456452301Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813847552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7064767488, + "inactive_split_bytes_peak": 8324517888, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:56.451880Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813847552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7060573184, + "inactive_split_bytes_peak": 8324517888, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:55.565472Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7064767488, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8324517888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T135456452301Z.pickle", + "timestamp_utc": "2026-08-01T13:54:56.555976Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T135456452301Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T135456452301Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f38874d3cc26da299286ff8710957e06c3b498f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T135456452301Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a8e94252b6479a70c6ce0660290b8595c6581c42bf8ad778a26300d388e8be1 +size 5609743 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T135457451675Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T135457451675Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c6d3731d8378a260aff244a1db101fc31ebfb8ef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T135457451675Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813847552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8324517888, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:57.451271Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813847552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7064767488, + "inactive_split_bytes_peak": 8324517888, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:56.571938Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8324517888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T135457451675Z.pickle", + "timestamp_utc": "2026-08-01T13:54:57.556581Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T135457451675Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T135457451675Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf0a1d73d4289ee41be6722f15df221ff6d35054 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T135457451675Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0aa2a27c0678339d9105e0cce36438fafed77a54208cdfa195257dbd3f5ccef9 +size 5615774 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T135458453836Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T135458453836Z.json new file mode 100644 index 0000000000000000000000000000000000000000..917ff5c71dccbfe23f68e791510cebe2041818a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T135458453836Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105813847552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7066864640, + "inactive_split_bytes_peak": 8324517888, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:58.453441Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105813847552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8324517888, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:57.572487Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7066864640, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8324517888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T135458453836Z.pickle", + "timestamp_utc": "2026-08-01T13:54:58.558355Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T135458453836Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T135458453836Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..68b7ab3379e1102e466ed5174fa60cf2fea5ff30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T135458453836Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3e0df11806bd99fca20d9b8d8cb8555f26c9b177712425e11df011ca6c6d68c +size 5609899 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T135551183416Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T135551183416Z.json new file mode 100644 index 0000000000000000000000000000000000000000..59e928c0b7e116ff9835b965bf3a72570c55b594 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T135551183416Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105813847552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 6792449024, + "inactive_split_bytes_peak": 8324517888, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:55:51.182946Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105813847552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105813847552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105813847552, + "device": 3, + "inactive_split_bytes_current": 7066864640, + "inactive_split_bytes_peak": 8324517888, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 108995280896, + "reserved_bytes_peak": 108995280896, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T13:54:58.574351Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6792449024, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 8324517888, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 108995280896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108995280896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T135551183416Z.pickle", + "timestamp_utc": "2026-08-01T13:55:51.392588Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T135551183416Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T135551183416Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd70364ee52717c0e38181d0c31cd479038418f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T135551183416Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec7528ef944bd03d9c3d3f183cf64343adaa98598ee78352bb01269583289180 +size 10204270 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T135449863303Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T135449863303Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e2a265b635cd84bad489216b7ca3e6aa8f5c74ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T135449863303Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855651328, + "active_bytes_current_delta": 31623753728, + "active_bytes_peak": 82174452736, + "active_bytes_peak_delta": 46707674112, + "after": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 82174452736, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 82174452736, + "device": 4, + "inactive_split_bytes_current": 6842464256, + "inactive_split_bytes_peak": 7236909056, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 98725527552, + "reserved_bytes_peak": 98725527552, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T13:54:49.862863Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855651328, + "allocated_bytes_current_delta": 31623753728, + "allocated_bytes_peak": 82174452736, + "allocated_bytes_peak_delta": 46707674112, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:53:43.245998Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6842464256, + "inactive_split_bytes_current_delta": 6710087680, + "inactive_split_bytes_peak": 7236909056, + "inactive_split_bytes_peak_delta": 7083499520, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 98725527552, + "reserved_bytes_current_delta": 59481522176, + "reserved_bytes_peak": 98725527552, + "reserved_bytes_peak_delta": 59481522176, + "segment_count_current": 502, + "segment_count_current_delta": 431, + "segment_count_peak": 502, + "segment_count_peak_delta": 431, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T135449863303Z.pickle", + "timestamp_utc": "2026-08-01T13:54:49.974922Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T135449863303Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T135449863303Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..761815835bf96bbf26a7aaee5ea4420ac792541e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T135449863303Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc6e5ed2566634890d7f0283420786eba6f6d1163cc038676a37a03a41d426e6 +size 5394569 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T135450901783Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T135450901783Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e09867d8c7b9d6e209b497a315240f4c0410b323 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T135450901783Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855651328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105810536448, + "active_bytes_peak_delta": 23636083712, + "after": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6624360448, + "inactive_split_bytes_peak": 7662941184, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:50.901352Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855651328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105810536448, + "allocated_bytes_peak_delta": 23636083712, + "before": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 82174452736, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 82174452736, + "device": 4, + "inactive_split_bytes_current": 6842464256, + "inactive_split_bytes_peak": 7236909056, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 98725527552, + "reserved_bytes_peak": 98725527552, + "segment_count_current": 502, + "segment_count_peak": 502, + "timestamp_utc": "2026-08-01T13:54:49.993724Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6624360448, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 7662941184, + "inactive_split_bytes_peak_delta": 426032128, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107684560896, + "reserved_bytes_current_delta": 8959033344, + "reserved_bytes_peak": 107684560896, + "reserved_bytes_peak_delta": 8959033344, + "segment_count_current": 534, + "segment_count_current_delta": 32, + "segment_count_peak": 534, + "segment_count_peak_delta": 32, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T135450901783Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.010786Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T135450901783Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T135450901783Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0099d0e00d44f37e5c669ae59bb3d8df0c6a3160 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T135450901783Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51b0738b2ca22343facc60c4d0fc0331d9a2e7bbc445a5035d6cd7094a4e6993 +size 5620200 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T135451886927Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T135451886927Z.json new file mode 100644 index 0000000000000000000000000000000000000000..02f61e665a102ebe3f7342dcb763edb185782a29 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T135451886927Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855651328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105810536448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6622263296, + "inactive_split_bytes_peak": 7794634752, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:51.886516Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855651328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105810536448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6624360448, + "inactive_split_bytes_peak": 7662941184, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:51.024321Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6622263296, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7794634752, + "inactive_split_bytes_peak_delta": 131693568, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107684560896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107684560896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 534, + "segment_count_current_delta": 0, + "segment_count_peak": 534, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T135451886927Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.986410Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T135451886927Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T135451886927Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9292225ec2ecda3a55986a4c51450ddf0ec3f64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T135451886927Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d80ffe73925e3d4bbea67ce44d3d6d968a58c346e098d85efe3f65b2c7caf65 +size 5611702 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T135452888374Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T135452888374Z.json new file mode 100644 index 0000000000000000000000000000000000000000..294f42c44b66b09370289d5feea3f41aa7ef5fde --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T135452888374Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855651328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105810536448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6857144320, + "inactive_split_bytes_peak": 8244698112, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:52.887930Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855651328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105810536448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6622263296, + "inactive_split_bytes_peak": 7794634752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:51.999104Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6857144320, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 8244698112, + "inactive_split_bytes_peak_delta": 450063360, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107684560896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107684560896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 534, + "segment_count_current_delta": 0, + "segment_count_peak": 534, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T135452888374Z.pickle", + "timestamp_utc": "2026-08-01T13:54:52.987210Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T135452888374Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T135452888374Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e2c9fa8ed3f3492efd3c51be7be9f0767847c42 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T135452888374Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6b25966f62273038ccda7c41b493a4e9070744bdbf105dee2c4abe55f525eed +size 5616399 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T135454422257Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T135454422257Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6672db5b1fe35c069d1e5621141094dc3187983a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T135454422257Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855651328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105810536448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6615971840, + "inactive_split_bytes_peak": 8244698112, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:54.421789Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855651328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105810536448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6857144320, + "inactive_split_bytes_peak": 8244698112, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:52.999849Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6615971840, + "inactive_split_bytes_current_delta": -241172480, + "inactive_split_bytes_peak": 8244698112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107684560896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107684560896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 534, + "segment_count_current_delta": 0, + "segment_count_peak": 534, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T135454422257Z.pickle", + "timestamp_utc": "2026-08-01T13:54:54.526003Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T135454422257Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T135454422257Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7014382b4ffe6dde384a2693c832f88d40b98e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T135454422257Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1482a0be81ab2afa049f9c6053f4339b71414b29f87290c10757a1d73e0f4f43 +size 5616112 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T135455442866Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T135455442866Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c424b4f2a03bc2ca2232a82a18bd2c82e2990104 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T135455442866Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855651328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105810536448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6848755712, + "inactive_split_bytes_peak": 8244698112, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:55.442445Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855651328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105810536448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6615971840, + "inactive_split_bytes_peak": 8244698112, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:54.539211Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6848755712, + "inactive_split_bytes_current_delta": 232783872, + "inactive_split_bytes_peak": 8244698112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107684560896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107684560896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 534, + "segment_count_current_delta": 0, + "segment_count_peak": 534, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T135455442866Z.pickle", + "timestamp_utc": "2026-08-01T13:54:55.547379Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T135455442866Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T135455442866Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5f572c19e48e1838e25e6d7dab0e4c904cf02ba0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T135455442866Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b06422328863c70e10eeb2f05c5d839a34acfe2f59eebe624d79b3b384236bc7 +size 5617363 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T135456452299Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T135456452299Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b71cd80c71fe8b606604bdc5d2da9d26b9b05217 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T135456452299Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855651328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105810536448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6618068992, + "inactive_split_bytes_peak": 8244698112, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:56.451871Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855651328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105810536448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6848755712, + "inactive_split_bytes_peak": 8244698112, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:55.565641Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6618068992, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 8244698112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107684560896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107684560896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 534, + "segment_count_current_delta": 0, + "segment_count_peak": 534, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T135456452299Z.pickle", + "timestamp_utc": "2026-08-01T13:54:56.552982Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T135456452299Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T135456452299Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9546a5f78071f2eeceea92df24512ede3fd5c3de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T135456452299Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc24c1e075e4c7e300a1d409ba64dd6101f5b2d7f10e11885dbf12706d9302d0 +size 5616089 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T135457451682Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T135457451682Z.json new file mode 100644 index 0000000000000000000000000000000000000000..810551b7433e8e86a1cf284d456a2ba5765afcb4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T135457451682Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855651328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105810536448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6615971840, + "inactive_split_bytes_peak": 8244698112, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:57.451256Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855651328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105810536448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6618068992, + "inactive_split_bytes_peak": 8244698112, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:56.565850Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6615971840, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8244698112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107684560896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107684560896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 534, + "segment_count_current_delta": 0, + "segment_count_peak": 534, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T135457451682Z.pickle", + "timestamp_utc": "2026-08-01T13:54:57.550777Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T135457451682Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T135457451682Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..310b89d82dc9f13b1d3aeb6e8da159ecc91c9be6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T135457451682Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee3a20c4dd655167fd7b0132c8b47251fa3bda3719d6c0d2cbb3853725d37cce +size 5616079 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T135458453869Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T135458453869Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c9afb6ea70214121b3a4f7336c03c206565b757 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T135458453869Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855651328, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105810536448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6855047168, + "inactive_split_bytes_peak": 8244698112, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:58.453438Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855651328, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105810536448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6615971840, + "inactive_split_bytes_peak": 8244698112, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:57.563544Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6855047168, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 8244698112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107684560896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107684560896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 534, + "segment_count_current_delta": 0, + "segment_count_peak": 534, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T135458453869Z.pickle", + "timestamp_utc": "2026-08-01T13:54:58.552437Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T135458453869Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T135458453869Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..171098e1d56182df1ab4b3fb6290646eae19852d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T135458453869Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4c5876c65e229bbd865ac22fb7a434e86131ef113d50c8ca2d8efa4dcb998f9 +size 5616304 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T135551183404Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T135551183404Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f46ea540788e13da19e4f7b3bd77b2cdc2d78a5d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T135551183404Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66805008384, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105810536448, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66805008384, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66805008384, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6345750528, + "inactive_split_bytes_peak": 8244698112, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:55:51.182921Z", + "world_size": 8 + }, + "allocated_bytes_current": 66805008384, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105810536448, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855651328, + "active_bytes_peak": 105810536448, + "allocated_bytes_current": 66855651328, + "allocated_bytes_peak": 105810536448, + "device": 4, + "inactive_split_bytes_current": 6855047168, + "inactive_split_bytes_peak": 8244698112, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 107684560896, + "reserved_bytes_peak": 107684560896, + "segment_count_current": 534, + "segment_count_peak": 534, + "timestamp_utc": "2026-08-01T13:54:58.565248Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6345750528, + "inactive_split_bytes_current_delta": -509296640, + "inactive_split_bytes_peak": 8244698112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 107684560896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 107684560896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 534, + "segment_count_current_delta": 0, + "segment_count_peak": 534, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T135551183404Z.pickle", + "timestamp_utc": "2026-08-01T13:55:51.377362Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T135551183404Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T135551183404Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..740edf63a9e308f8598262772beb2789d6864b43 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T135551183404Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e239c4eab6e26e15ecb5ebb674665b5fdc15132e69571ffecb3e0426c7c42901 +size 10204561 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T135449863337Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T135449863337Z.json new file mode 100644 index 0000000000000000000000000000000000000000..666af06923a3c68b733208eff1941dae051f3134 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T135449863337Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 82172224512, + "active_bytes_peak_delta": 46705888256, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 82172224512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 82172224512, + "device": 5, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_peak": 7289163264, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 99216261120, + "reserved_bytes_peak": 99216261120, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:54:49.862752Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 82172224512, + "allocated_bytes_peak_delta": 46705888256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:53:43.246146Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_current_delta": 7156343808, + "inactive_split_bytes_peak": 7289163264, + "inactive_split_bytes_peak_delta": 7128777216, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 99216261120, + "reserved_bytes_current_delta": 59972255744, + "reserved_bytes_peak": 99216261120, + "reserved_bytes_peak_delta": 59972255744, + "segment_count_current": 505, + "segment_count_current_delta": 434, + "segment_count_peak": 505, + "segment_count_peak_delta": 434, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T135449863337Z.pickle", + "timestamp_utc": "2026-08-01T13:54:49.973585Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T135449863337Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T135449863337Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..647f7a1346eae401ec090e890a95e73d701e5750 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T135449863337Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:508691de52db6a0b592ec0f616225c378c8f7ced96aefdc19b1d197b51c09373 +size 5385517 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T135450901770Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T135450901770Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dc99652b781ff15e423be3136a4c000bbc05aa62 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T135450901770Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 23643720192, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_peak": 7423191040, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:50.901375Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 23643720192, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 82172224512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 82172224512, + "device": 5, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_peak": 7289163264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 99216261120, + "reserved_bytes_peak": 99216261120, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:54:49.992255Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 7423191040, + "inactive_split_bytes_peak_delta": 134027776, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108525518848, + "reserved_bytes_current_delta": 9309257728, + "reserved_bytes_peak": 108525518848, + "reserved_bytes_peak_delta": 9309257728, + "segment_count_current": 537, + "segment_count_current_delta": 32, + "segment_count_peak": 537, + "segment_count_peak_delta": 32, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T135450901770Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.011679Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T135450901770Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T135450901770Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3444e3fb020fe8d3f9acfa14266d833221333e66 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T135450901770Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1ec7e663b02c62436ee9d62b7118255f4ba48d515f6d061856bbfda018780d0 +size 5618537 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T135451886917Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T135451886917Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db697a846bc829e86da1162c11b4cf14bd202c52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T135451886917Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 7927024640, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:51.886529Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_peak": 7423191040, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:51.025125Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7927024640, + "inactive_split_bytes_peak_delta": 503833600, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108525518848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108525518848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T135451886917Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.987673Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T135451886917Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T135451886917Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..83238ba5cac660877f8515cf557efe4ae7ebb6ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T135451886917Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fa7ca99e0fdf96349ae4a4c03d8c26f92ae0d71193b1916a635f2d6430b70bc +size 5610725 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T135452888359Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T135452888359Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a00810356dbc3c010fa8461e6b4314eddeee95d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T135452888359Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7303842816, + "inactive_split_bytes_peak": 7927024640, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:52.887954Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 7927024640, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:52.002309Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7303842816, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 7927024640, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108525518848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108525518848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T135452888359Z.pickle", + "timestamp_utc": "2026-08-01T13:54:52.989066Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T135452888359Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T135452888359Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6cf75e74f3414fed301bd5b41b935f19fc5f2223 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T135452888359Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cefbc202a6c2fccc7efe3f9f8b08559aef31f4f23679badfe2359a6a0a2f13fc +size 5612078 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T135454422201Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T135454422201Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a5c882a91c363299442f00ffdbf739a7d4d0f0a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T135454422201Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8037125120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:54.421780Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7303842816, + "inactive_split_bytes_peak": 7927024640, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:53.001902Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_current_delta": -241172480, + "inactive_split_bytes_peak": 8037125120, + "inactive_split_bytes_peak_delta": 110100480, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108525518848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108525518848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T135454422201Z.pickle", + "timestamp_utc": "2026-08-01T13:54:54.526296Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T135454422201Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T135454422201Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cbe5e942a2a3cfbb7d85feb8af3b6e9d283dbef6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T135454422201Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d27739b8051dc7c79afd859091b7f448c227c96879d56f5634fefd1aa4355f8 +size 5610528 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T135455442859Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T135455442859Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc289504bc8426f6a31ae3680251b3e8028ea26c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T135455442859Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7295454208, + "inactive_split_bytes_peak": 8037125120, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:55.442448Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8037125120, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:54.539361Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7295454208, + "inactive_split_bytes_current_delta": 232783872, + "inactive_split_bytes_peak": 8037125120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108525518848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108525518848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T135455442859Z.pickle", + "timestamp_utc": "2026-08-01T13:54:55.541883Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T135455442859Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T135455442859Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca3827169096d8de44ed7d65cc4171eed93f3a50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T135455442859Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74e4200cc2cfbcf06203410c60c4f8e87e5eb62d0067d3203d3f199a3c3509c8 +size 5616406 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T135456452287Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T135456452287Z.json new file mode 100644 index 0000000000000000000000000000000000000000..912496bab0c1b40309c4a49e030f367adeefa7a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T135456452287Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7299648512, + "inactive_split_bytes_peak": 8272006144, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:56.451882Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7295454208, + "inactive_split_bytes_peak": 8037125120, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:55.554899Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7299648512, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8272006144, + "inactive_split_bytes_peak_delta": 234881024, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108525518848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108525518848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T135456452287Z.pickle", + "timestamp_utc": "2026-08-01T13:54:56.555416Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T135456452287Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T135456452287Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..141e8fedde582e9807985e8472fcd84a65ca32d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T135456452287Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e53f06dc359527f7a73164e112b5cb08f748e7ebfb456660145b21c2653a9c1 +size 5610270 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T135457451685Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T135457451685Z.json new file mode 100644 index 0000000000000000000000000000000000000000..daf2868d935192790d8cdccacaf677114c11c8ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T135457451685Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8272006144, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:57.451282Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7299648512, + "inactive_split_bytes_peak": 8272006144, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:56.571031Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 8272006144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108525518848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108525518848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T135457451685Z.pickle", + "timestamp_utc": "2026-08-01T13:54:57.551396Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T135457451685Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T135457451685Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..99f19d68ba083127403c5c50f3fb1ad83b3eea81 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T135457451685Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21b1de6ec242400df8ac2dab9e39135f333ad4b053df1b182e9c376b380157c2 +size 5616262 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T135458453865Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T135458453865Z.json new file mode 100644 index 0000000000000000000000000000000000000000..716ca78cf90339c72e8519b949a95e151ea71fd6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T135458453865Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7301745664, + "inactive_split_bytes_peak": 8274103296, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:58.453457Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8272006144, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:57.564276Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7301745664, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 8274103296, + "inactive_split_bytes_peak_delta": 2097152, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108525518848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108525518848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T135458453865Z.pickle", + "timestamp_utc": "2026-08-01T13:54:58.556796Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T135458453865Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T135458453865Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b2ceab6934d1b37d584e2c833a7c3ab66a0d513 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T135458453865Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1f275faec1cbb63ca141ad2652cda26da67f13f85d695b183501d387cdb449c +size 5610429 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T135551183403Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T135551183403Z.json new file mode 100644 index 0000000000000000000000000000000000000000..14bd0a8b5305ee0ed369c8411abd8820a354d217 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T135551183403Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105815944704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 6792449024, + "inactive_split_bytes_peak": 8274103296, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:55:51.182927Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105815944704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105815944704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105815944704, + "device": 5, + "inactive_split_bytes_current": 7301745664, + "inactive_split_bytes_peak": 8274103296, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 108525518848, + "reserved_bytes_peak": 108525518848, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:58.569863Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6792449024, + "inactive_split_bytes_current_delta": -509296640, + "inactive_split_bytes_peak": 8274103296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 108525518848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108525518848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T135551183403Z.pickle", + "timestamp_utc": "2026-08-01T13:55:51.376672Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T135551183403Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T135551183403Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..184b174453e97fbbbaf13b157ad6b80b978f8dcd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T135551183403Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63916a8e6efb1b682e24d83c187e0605b435d1da7ec8064b2409fe08a20821c8 +size 10204817 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T135449863292Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T135449863292Z.json new file mode 100644 index 0000000000000000000000000000000000000000..46d15fce019c7df6b506b6049ef37466c7acc039 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T135449863292Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 82172224512, + "active_bytes_peak_delta": 46705888256, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 82172224512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 82172224512, + "device": 6, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_peak": 7289163264, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 99216261120, + "reserved_bytes_peak": 99216261120, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:54:49.862712Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 82172224512, + "allocated_bytes_peak_delta": 46705888256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:53:43.246199Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_current_delta": 7156343808, + "inactive_split_bytes_peak": 7289163264, + "inactive_split_bytes_peak_delta": 7128777216, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 99216261120, + "reserved_bytes_current_delta": 59972255744, + "reserved_bytes_peak": 99216261120, + "reserved_bytes_peak_delta": 59972255744, + "segment_count_current": 505, + "segment_count_current_delta": 434, + "segment_count_peak": 505, + "segment_count_peak_delta": 434, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T135449863292Z.pickle", + "timestamp_utc": "2026-08-01T13:54:49.975985Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T135449863292Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T135449863292Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..905770b5f67de81383ad90053ed43238a8e68020 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T135449863292Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d1ce5d28c3888d876d23186adab047f3ebc3891b1745b1daeb0b03c8ff9d556 +size 5385509 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T135450901795Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T135450901795Z.json new file mode 100644 index 0000000000000000000000000000000000000000..508ad0dc8c09f7d88d26f953b29cb3e71347992e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T135450901795Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105811991552, + "active_bytes_peak_delta": 23639767040, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105811991552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105811991552, + "device": 6, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_peak": 8040208384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108173197312, + "reserved_bytes_peak": 108173197312, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T13:54:50.901396Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105811991552, + "allocated_bytes_peak_delta": 23639767040, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 82172224512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 82172224512, + "device": 6, + "inactive_split_bytes_current": 7289162752, + "inactive_split_bytes_peak": 7289163264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 99216261120, + "reserved_bytes_peak": 99216261120, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:54:49.997435Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 8040208384, + "inactive_split_bytes_peak_delta": 751045120, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108173197312, + "reserved_bytes_current_delta": 8956936192, + "reserved_bytes_peak": 108173197312, + "reserved_bytes_peak_delta": 8956936192, + "segment_count_current": 536, + "segment_count_current_delta": 31, + "segment_count_peak": 536, + "segment_count_peak_delta": 31, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T135450901795Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.016584Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T135450901795Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T135450901795Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..67d0e91a5768c49ecb3df9e543c265b72977acb1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T135450901795Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17be2f161b9774e690c88d2624d60d05aa54882c420b3ad3e46d580b1d350a83 +size 5617989 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T135451886884Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T135451886884Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dce026830741a0dde7e1240ef70ec630d542e1db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T135451886884Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105811991552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105811991552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105811991552, + "device": 6, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 8040208384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:51.886489Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105811991552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105811991552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105811991552, + "device": 6, + "inactive_split_bytes_current": 7071058944, + "inactive_split_bytes_peak": 8040208384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108173197312, + "reserved_bytes_peak": 108173197312, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T13:54:51.034844Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8040208384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 413138944, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 413138944, + "segment_count_current": 537, + "segment_count_current_delta": 1, + "segment_count_peak": 537, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T135451886884Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.992952Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T135451886884Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T135451886884Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6afe07f8cdefc28d886ab5d8a232fd4ab6a1def4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T135451886884Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f88f67c8ac23975b558b0852999af3a16307f6e7841074dcdfeb513837c9514 +size 5610211 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T135452888317Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T135452888317Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b5b3844c24e9f823b6beb38636bac1f68a5a4b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T135452888317Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105811991552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105811991552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105811991552, + "device": 6, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 8336583680, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:52.887910Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105811991552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105811991552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105811991552, + "device": 6, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 8040208384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:52.008937Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8336583680, + "inactive_split_bytes_peak_delta": 296375296, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T135452888317Z.pickle", + "timestamp_utc": "2026-08-01T13:54:52.992229Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T135452888317Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T135452888317Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dfdfe9daaa2d29207e1f0ec05693445c932d7c30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T135452888317Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c90efe442c5687fd343d7e0115604989fd1defcbdeea41acd98c54ddf7744279 +size 5611828 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T135454422166Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T135454422166Z.json new file mode 100644 index 0000000000000000000000000000000000000000..30053ec086ff60b29e7498bddc9fc1c32bc26092 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T135454422166Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105811991552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105811991552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105811991552, + "device": 6, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8336583680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:54.421750Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105811991552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105811991552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105811991552, + "device": 6, + "inactive_split_bytes_current": 7068961792, + "inactive_split_bytes_peak": 8336583680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:53.008088Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 8336583680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T135454422166Z.pickle", + "timestamp_utc": "2026-08-01T13:54:54.528608Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T135454422166Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T135454422166Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b16da139419d1f3271cb1ad8ad2791fbe0b60c2b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T135454422166Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fac3e7a571e66a0dca353993f56b6297b6c1ad966127422f851034d49f865c2 +size 5610302 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T135455442973Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T135455442973Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f4370894a00ec8fa9464d69f02259f5f70d40909 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T135455442973Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105816366080, + "active_bytes_peak_delta": 4374528, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105816366080, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105816366080, + "device": 6, + "inactive_split_bytes_current": 7060573184, + "inactive_split_bytes_peak": 8336583680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:55.442493Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105816366080, + "allocated_bytes_peak_delta": 4374528, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105811991552, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105811991552, + "device": 6, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8336583680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:54.544519Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7060573184, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8336583680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T135455442973Z.pickle", + "timestamp_utc": "2026-08-01T13:54:55.551483Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T135455442973Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T135455442973Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c1cc065b6fae4ea98b8cf660238518806a7a9be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T135455442973Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:735c9f7859dd9c3d30a0386cbb7653e64dbd820fcfc14954c961997aee967fd9 +size 5615949 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T135456452354Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T135456452354Z.json new file mode 100644 index 0000000000000000000000000000000000000000..831832e7d08a10c0fe0692eb8dd776666aaa5182 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T135456452354Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105816366080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105816366080, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105816366080, + "device": 6, + "inactive_split_bytes_current": 7064767488, + "inactive_split_bytes_peak": 8336583680, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:56.451871Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105816366080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105816366080, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105816366080, + "device": 6, + "inactive_split_bytes_current": 7060573184, + "inactive_split_bytes_peak": 8336583680, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:55.567770Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7064767488, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8336583680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T135456452354Z.pickle", + "timestamp_utc": "2026-08-01T13:54:56.558115Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T135456452354Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T135456452354Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bae44b1b4d15af1aafe4fe74b7ed2e06e3579b3f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T135456452354Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a532942df04c420e8736740198b1bc6053f52b1c5ceac15f4716e03cdd2088b +size 5609889 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T135457451759Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T135457451759Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f2fd069d4fd2d80b92fff6ee1587003e9dc1bc9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T135457451759Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105816366080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105816366080, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105816366080, + "device": 6, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8336583680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:57.451286Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105816366080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105816366080, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105816366080, + "device": 6, + "inactive_split_bytes_current": 7064767488, + "inactive_split_bytes_peak": 8336583680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:56.574186Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8336583680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T135457451759Z.pickle", + "timestamp_utc": "2026-08-01T13:54:57.556983Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T135457451759Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T135457451759Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d2223f186704c98c6ffa48c6beadebfc53ced1e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T135457451759Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a3abd27435da4bc081c0926bea28ecf7a26e44c9ed9d673bd8f33f356c419ab +size 5615848 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T135458453909Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T135458453909Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e0428a97e16b7ddb260635a06d66f2d8dba56535 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T135458453909Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105816366080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105816366080, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105816366080, + "device": 6, + "inactive_split_bytes_current": 7066864640, + "inactive_split_bytes_peak": 8336583680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:58.453439Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105816366080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105816366080, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105816366080, + "device": 6, + "inactive_split_bytes_current": 7062670336, + "inactive_split_bytes_peak": 8336583680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:57.572826Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7066864640, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8336583680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T135458453909Z.pickle", + "timestamp_utc": "2026-08-01T13:54:58.558859Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T135458453909Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T135458453909Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f8a4a54c0f4654872e5b3753e71b9b57c14de5b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T135458453909Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4641b4f8113ea56fa59ab5e5b0df78a88e0f0e16935f2112955c4c7fb0d0a002 +size 5609984 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T135551183416Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T135551183416Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c530ed07921ee0d24e86d8b7cc1ffb90d9aca13f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T135551183416Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105816366080, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 105816366080, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 105816366080, + "device": 6, + "inactive_split_bytes_current": 6792449024, + "inactive_split_bytes_peak": 8336583680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:55:51.182933Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105816366080, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 105816366080, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 105816366080, + "device": 6, + "inactive_split_bytes_current": 7066864640, + "inactive_split_bytes_peak": 8336583680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:58.574612Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6792449024, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 8336583680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T135551183416Z.pickle", + "timestamp_utc": "2026-08-01T13:55:51.380042Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T135551183416Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T135551183416Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9eb3847c273951da2efe5529b883307d7d686b4a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T135551183416Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c374ccd71c20832749fa7f5980f176288bea611bc4e2801b53ebf1c0f4fa1c2 +size 10204382 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T135449863158Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T135449863158Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f4d98343ec064d698760c1890e4c2aba05564dd9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T135449863158Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 31622094336, + "active_bytes_peak": 82172224512, + "active_bytes_peak_delta": 46705888256, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 82172224512, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 82172224512, + "device": 7, + "inactive_split_bytes_current": 7289162240, + "inactive_split_bytes_peak": 7289163264, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 99216261120, + "reserved_bytes_peak": 99216261120, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:54:49.862691Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 31622094336, + "allocated_bytes_peak": 82172224512, + "allocated_bytes_peak_delta": 46705888256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:53:43.246337Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7289162240, + "inactive_split_bytes_current_delta": 7156343296, + "inactive_split_bytes_peak": 7289163264, + "inactive_split_bytes_peak_delta": 7128777216, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 99216261120, + "reserved_bytes_current_delta": 59972255744, + "reserved_bytes_peak": 99216261120, + "reserved_bytes_peak_delta": 59972255744, + "segment_count_current": 505, + "segment_count_current_delta": 434, + "segment_count_peak": 505, + "segment_count_peak_delta": 434, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T135449863158Z.pickle", + "timestamp_utc": "2026-08-01T13:54:49.971388Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T135449863158Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T135449863158Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4471a635873e3023bad6bdf29e34479affb90431 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T135449863158Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9887bdc4caf07764043a0b8fbaf42d2ed952cae1c63881d3e86eea7bb39244ae +size 5387118 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T135450901748Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T135450901748Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8dcba63403a9fc2f8bb59dfa57678583702683ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T135450901748Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814503424, + "active_bytes_peak_delta": 23642278912, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7071058432, + "inactive_split_bytes_peak": 7958357504, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108173197312, + "reserved_bytes_peak": 108173197312, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T13:54:50.901325Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814503424, + "allocated_bytes_peak_delta": 23642278912, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 82172224512, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 82172224512, + "device": 7, + "inactive_split_bytes_current": 7289162240, + "inactive_split_bytes_peak": 7289163264, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 99216261120, + "reserved_bytes_peak": 99216261120, + "segment_count_current": 505, + "segment_count_peak": 505, + "timestamp_utc": "2026-08-01T13:54:49.989627Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7071058432, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 7958357504, + "inactive_split_bytes_peak_delta": 669194240, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108173197312, + "reserved_bytes_current_delta": 8956936192, + "reserved_bytes_peak": 108173197312, + "reserved_bytes_peak_delta": 8956936192, + "segment_count_current": 536, + "segment_count_current_delta": 31, + "segment_count_peak": 536, + "segment_count_peak_delta": 31, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T135450901748Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.010994Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T135450901748Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T135450901748Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fee65bb180072d1347907b8b265c062d8d177158 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T135450901748Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee3f5e06b157a31f5263892ce8777564fe8c1fa664d536ab35fcbb77f6f8500a +size 5619729 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T135451886891Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T135451886891Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4aadbd9aeb9fb25bd07ee37f0511806cf5eba706 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T135451886891Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814503424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7068961280, + "inactive_split_bytes_peak": 7988559360, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:51.886493Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814503424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7071058432, + "inactive_split_bytes_peak": 7958357504, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108173197312, + "reserved_bytes_peak": 108173197312, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T13:54:51.024439Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7068961280, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7988559360, + "inactive_split_bytes_peak_delta": 30201856, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 413138944, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 413138944, + "segment_count_current": 537, + "segment_count_current_delta": 1, + "segment_count_peak": 537, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T135451886891Z.pickle", + "timestamp_utc": "2026-08-01T13:54:51.988500Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T135451886891Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T135451886891Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..37b21f38e4ec13151ce82f0ec429d8a2d61f686a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T135451886891Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ae4f497dd3a27b3d81e9370c46897b08fc88fcf6adfa3e959e945c65e07599f +size 5611846 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T135452888331Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T135452888331Z.json new file mode 100644 index 0000000000000000000000000000000000000000..02c2d1ef412e7484af0f1d9aa993a1c9aa09190a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T135452888331Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814503424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7303842304, + "inactive_split_bytes_peak": 7988559360, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:52.887910Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814503424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7068961280, + "inactive_split_bytes_peak": 7988559360, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:52.001463Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7303842304, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 7988559360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T135452888331Z.pickle", + "timestamp_utc": "2026-08-01T13:54:52.988964Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T135452888331Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T135452888331Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6fc3de27c138b2a0bb3f039714b0d299af8e4823 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T135452888331Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae57e5409b9813d6c8d8af1b398705bef0e7bb01dabdff3986103bac0c8e035b +size 5613493 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T135454422244Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T135454422244Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e7ebbfaa14d690ffc39d9b847bb48e699055f5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T135454422244Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814503424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7062669824, + "inactive_split_bytes_peak": 7988559360, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:54.421808Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814503424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7303842304, + "inactive_split_bytes_peak": 7988559360, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:53.001846Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7062669824, + "inactive_split_bytes_current_delta": -241172480, + "inactive_split_bytes_peak": 7988559360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T135454422244Z.pickle", + "timestamp_utc": "2026-08-01T13:54:54.526194Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T135454422244Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T135454422244Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..84084ed4d097488407529472b3167dd71af51e7f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T135454422244Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:810d5a1ff6e02fecf3788e109c19ce2251425a8fa4c5b2537b9ff1780cfeede7 +size 5611899 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T135455442821Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T135455442821Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c2742b7cc61af734c38e3ff7aa4a1fe6a0aa708e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T135455442821Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814503424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7295453696, + "inactive_split_bytes_peak": 7988559360, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:55.442413Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814503424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7062669824, + "inactive_split_bytes_peak": 7988559360, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:54.539302Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7295453696, + "inactive_split_bytes_current_delta": 232783872, + "inactive_split_bytes_peak": 7988559360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T135455442821Z.pickle", + "timestamp_utc": "2026-08-01T13:54:55.543318Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T135455442821Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T135455442821Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cad82962b42a3fc5f50c1178b08701843bf68a08 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T135455442821Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9728f68e80a8c2ad43f76fcf27adcd39aba99b933fec9a000b8ef4df9d0735d6 +size 5617986 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T135456452307Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T135456452307Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d6cf988113106f7f2cb79233a3d58a12990f0fcb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T135456452307Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814503424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7064766976, + "inactive_split_bytes_peak": 7988559360, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:56.451884Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814503424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7295453696, + "inactive_split_bytes_peak": 7988559360, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:55.556200Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7064766976, + "inactive_split_bytes_current_delta": -230686720, + "inactive_split_bytes_peak": 7988559360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T135456452307Z.pickle", + "timestamp_utc": "2026-08-01T13:54:56.551897Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T135456452307Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T135456452307Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e8aa9ef88c47829bf2a13cd76534589b5d66aef3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T135456452307Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3770da94b75a1c9b1ba6463d2877a679122d557a1a1206924f2ee6b33290241a +size 5611856 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T135457451679Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T135457451679Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd0bee174b0647de3afe2afd3e1a73f499954d58 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T135457451679Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814503424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7062669824, + "inactive_split_bytes_peak": 7988559360, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:57.451285Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814503424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7064766976, + "inactive_split_bytes_peak": 7988559360, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:56.564544Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7062669824, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 7988559360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T135457451679Z.pickle", + "timestamp_utc": "2026-08-01T13:54:57.550074Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T135457451679Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T135457451679Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..024b8d2be1c91718905aa3d54bbe1b3d62537776 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T135457451679Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1b217d7508bbd2f89e2de294b1319c948f38ef69ed76ae88e0ce198e7036776 +size 5617913 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T135458453862Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T135458453862Z.json new file mode 100644 index 0000000000000000000000000000000000000000..00e563bf139c0db400570809345be582e2df0cbe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T135458453862Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 105814503424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7301745152, + "inactive_split_bytes_peak": 7988559360, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:58.453467Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105814503424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7062669824, + "inactive_split_bytes_peak": 7988559360, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:57.562506Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7301745152, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 7988559360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T135458453862Z.pickle", + "timestamp_utc": "2026-08-01T13:54:58.551147Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T135458453862Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T135458453862Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6fed0b118810c1787eb7e690a848ce91ff57dcc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T135458453862Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e44a5518fe746fe2f8d95476d72103201f36010628c694af4ac67b97f966ca8 +size 5612086 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T135551183410Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T135551183410Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a911136a59e348e95fc4292ca8c148fe0a1482bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T135551183410Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906624, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 105814503424, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906624, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66802906624, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 6792448512, + "inactive_split_bytes_peak": 7988559360, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:55:51.182939Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906624, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 105814503424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 105814503424, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 105814503424, + "device": 7, + "inactive_split_bytes_current": 7301745152, + "inactive_split_bytes_peak": 7988559360, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 108586336256, + "reserved_bytes_peak": 108586336256, + "segment_count_current": 537, + "segment_count_peak": 537, + "timestamp_utc": "2026-08-01T13:54:58.563744Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6792448512, + "inactive_split_bytes_current_delta": -509296640, + "inactive_split_bytes_peak": 7988559360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 108586336256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 108586336256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 537, + "segment_count_current_delta": 0, + "segment_count_peak": 537, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T135551183410Z.pickle", + "timestamp_utc": "2026-08-01T13:55:51.377655Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T135551183410Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T135551183410Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9eb06837fd7161aaf5b41f0a193bd709f8b3593a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T135551183410Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2accad48ac2ff5a938b591b1e08df768eaba8a27a6f4cee0a4d5b8f0fdbaae0 +size 10206440 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..2a8615d1a89e0c7cd10d6ab87cfe88a546c9ff3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 2048 +capacity: 1.50 +expert_dispatch_overlap: on +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6035 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 225 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T135837143055Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T135837143055Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b23d6327e1cc0dfb6ec132367f9eae002ec9389 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T135837143055Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 37208849408, + "active_bytes_peak": 83124963328, + "active_bytes_peak_delta": 47658627072, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 83124963328, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 82146322432, + "device": 0, + "inactive_split_bytes_current": 6618131456, + "inactive_split_bytes_peak": 7675571200, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 100506009600, + "reserved_bytes_peak": 100506009600, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T13:58:37.142491Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 37208849408, + "allocated_bytes_peak": 82146322432, + "allocated_bytes_peak_delta": 46679986176, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T13:57:29.296680Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6618131456, + "inactive_split_bytes_current_delta": 6483215360, + "inactive_split_bytes_peak": 7675571200, + "inactive_split_bytes_peak_delta": 7514939392, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 100506009600, + "reserved_bytes_current_delta": 61259907072, + "reserved_bytes_peak": 100506009600, + "reserved_bytes_peak_delta": 61259907072, + "segment_count_current": 467, + "segment_count_current_delta": 395, + "segment_count_peak": 467, + "segment_count_peak_delta": 395, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T135837143055Z.pickle", + "timestamp_utc": "2026-08-01T13:58:37.265120Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T135837143055Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T135837143055Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ea52392a5b06a54037018bcc9bd8c9a7efa3375 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T135837143055Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47019cf00d3b7ff66633aadc84ef3318c43fb07eac0c7e9e8ebb7663d3aa5bab +size 5951925 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T135838280860Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T135838280860Z.json new file mode 100644 index 0000000000000000000000000000000000000000..637c08319ebf3e9bbf037fb84095e6370ff763a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T135838280860Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418180608, + "active_bytes_peak_delta": 23293217280, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105787577344, + "device": 0, + "inactive_split_bytes_current": 5909294080, + "inactive_split_bytes_peak": 8093563904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 110882717696, + "reserved_bytes_peak": 110882717696, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T13:58:38.280443Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787577344, + "allocated_bytes_peak_delta": 23641254912, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 83124963328, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 82146322432, + "device": 0, + "inactive_split_bytes_current": 6618131456, + "inactive_split_bytes_peak": 7675571200, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 100506009600, + "reserved_bytes_peak": 100506009600, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T13:58:37.284277Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5909294080, + "inactive_split_bytes_current_delta": -708837376, + "inactive_split_bytes_peak": 8093563904, + "inactive_split_bytes_peak_delta": 417992704, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 110882717696, + "reserved_bytes_current_delta": 10376708096, + "reserved_bytes_peak": 110882717696, + "reserved_bytes_peak_delta": 10376708096, + "segment_count_current": 500, + "segment_count_current_delta": 33, + "segment_count_peak": 500, + "segment_count_peak_delta": 33, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T135838280860Z.pickle", + "timestamp_utc": "2026-08-01T13:58:38.400394Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T135838280860Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T135838280860Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..43c1b0b332ba705a870ecdfe8df01852cea106a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T135838280860Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc7b2687903dda9aee08c8b01748e362b6a5c1e298a68e93bd09ce209edb2cdd +size 6122993 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T135839419371Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T135839419371Z.json new file mode 100644 index 0000000000000000000000000000000000000000..075d20e69944e8b00faf8a594976d321ae6a1629 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T135839419371Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418180608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105787577344, + "device": 0, + "inactive_split_bytes_current": 5942848512, + "inactive_split_bytes_peak": 8494368768, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:39.418959Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787577344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105787577344, + "device": 0, + "inactive_split_bytes_current": 5909294080, + "inactive_split_bytes_peak": 8093563904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 110882717696, + "reserved_bytes_peak": 110882717696, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T13:58:38.416668Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5942848512, + "inactive_split_bytes_current_delta": 33554432, + "inactive_split_bytes_peak": 8494368768, + "inactive_split_bytes_peak_delta": 400804864, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_current_delta": 1235222528, + "reserved_bytes_peak": 112117940224, + "reserved_bytes_peak_delta": 1235222528, + "segment_count_current": 503, + "segment_count_current_delta": 3, + "segment_count_peak": 503, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T135839419371Z.pickle", + "timestamp_utc": "2026-08-01T13:58:39.534973Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T135839419371Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T135839419371Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..099db09d594aaab2cc6044df19275317345408e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T135839419371Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:182f75dac388bfac20c2cfba32312b67b657355e485ae4b26ced440744aab295 +size 6625608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T135840796941Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T135840796941Z.json new file mode 100644 index 0000000000000000000000000000000000000000..39aa6a56aeaf72615f0055dbdbf91d6e3e724090 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T135840796941Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418180608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105787577344, + "device": 0, + "inactive_split_bytes_current": 6303558656, + "inactive_split_bytes_peak": 8494368768, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:40.796541Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787577344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105787577344, + "device": 0, + "inactive_split_bytes_current": 5942848512, + "inactive_split_bytes_peak": 8494368768, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:39.551291Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6303558656, + "inactive_split_bytes_current_delta": 360710144, + "inactive_split_bytes_peak": 8494368768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112117940224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T135840796941Z.pickle", + "timestamp_utc": "2026-08-01T13:58:40.911428Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T135840796941Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T135840796941Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..079232e0164fdf3a20e50b3a5f970dddea4d2227 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T135840796941Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d5e128600f3d18f5acc374dedb5648c9fbcba5632ea2d88c33aa3257757b14d +size 6621779 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T135842181503Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T135842181503Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f1753616d37ea9f3a16cd95b33e9cdefdcb265cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T135842181503Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418180608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105787577344, + "device": 0, + "inactive_split_bytes_current": 6265809920, + "inactive_split_bytes_peak": 8494368768, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:42.181069Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787577344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105787577344, + "device": 0, + "inactive_split_bytes_current": 6303558656, + "inactive_split_bytes_peak": 8494368768, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:40.927353Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6265809920, + "inactive_split_bytes_current_delta": -37748736, + "inactive_split_bytes_peak": 8494368768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112117940224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T135842181503Z.pickle", + "timestamp_utc": "2026-08-01T13:58:42.293634Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T135842181503Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T135842181503Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f8e643165b135f0a94b8db07231e7d3bec588b79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T135842181503Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:351629c1e13df204aa9792d176ab35fe7e0577591e70fe27cfa0be802b768e40 +size 6621746 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T135843321316Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T135843321316Z.json new file mode 100644 index 0000000000000000000000000000000000000000..587226718525ad29116be8162ec8518ba6fce0cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T135843321316Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418180608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105787577344, + "device": 0, + "inactive_split_bytes_current": 6060289024, + "inactive_split_bytes_peak": 8894116864, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:43.320903Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787577344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105787577344, + "device": 0, + "inactive_split_bytes_current": 6265809920, + "inactive_split_bytes_peak": 8494368768, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:42.309174Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6060289024, + "inactive_split_bytes_current_delta": -205520896, + "inactive_split_bytes_peak": 8894116864, + "inactive_split_bytes_peak_delta": 399748096, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112117940224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T135843321316Z.pickle", + "timestamp_utc": "2026-08-01T13:58:43.431554Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T135843321316Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T135843321316Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..996a1f36eda613f131804557daa54a4d76a014b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T135843321316Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d142c95e28c50b61578b9b9817d8e68d4782c346b57c0984cd345d0723f3a51 +size 6639422 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T135844462055Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T135844462055Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36549b7e3b82e1664b7a03f1a6c067c943c0ed88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T135844462055Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418180608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105787577344, + "device": 0, + "inactive_split_bytes_current": 6064483328, + "inactive_split_bytes_peak": 8894116864, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:44.461658Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787577344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105787577344, + "device": 0, + "inactive_split_bytes_current": 6060289024, + "inactive_split_bytes_peak": 8894116864, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:43.447266Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6064483328, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8894116864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112117940224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T135844462055Z.pickle", + "timestamp_utc": "2026-08-01T13:58:44.572456Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T135844462055Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T135844462055Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a70c3add607aa6810e776a99a84dc7ce4aeeb3e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T135844462055Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f11d32d28791922cf727b5bb3b7c11a8ea26436bbef942eeafd9f679b7d3e388 +size 6621486 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T135845749056Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T135845749056Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6a22c51b698836bb7fe5227bf7e4a40273d7bd28 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T135845749056Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418180608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105788905472, + "device": 0, + "inactive_split_bytes_current": 6853012480, + "inactive_split_bytes_peak": 8894116864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:45.748646Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788905472, + "allocated_bytes_peak_delta": 1328128, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105787577344, + "device": 0, + "inactive_split_bytes_current": 6064483328, + "inactive_split_bytes_peak": 8894116864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:44.588036Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6853012480, + "inactive_split_bytes_current_delta": 788529152, + "inactive_split_bytes_peak": 8894116864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112117940224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T135845749056Z.pickle", + "timestamp_utc": "2026-08-01T13:58:45.857949Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T135845749056Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T135845749056Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..543f50c10ae325efb16ead72b42ae41b106a4ddb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T135845749056Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e93a4485888f0ee7b85fa2d5838c8bafd3b6061a5df8cae7c2e9736b79d637a +size 6620734 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T135846884628Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T135846884628Z.json new file mode 100644 index 0000000000000000000000000000000000000000..829263c5699ae1b74023c15e783f886395b393e1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T135846884628Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440304640, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418180608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105788905472, + "device": 0, + "inactive_split_bytes_current": 6326627328, + "inactive_split_bytes_peak": 8894116864, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:46.884223Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440304640, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788905472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105788905472, + "device": 0, + "inactive_split_bytes_current": 6853012480, + "inactive_split_bytes_peak": 8894116864, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:45.873441Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6326627328, + "inactive_split_bytes_current_delta": -526385152, + "inactive_split_bytes_peak": 8894116864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112117940224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T135846884628Z.pickle", + "timestamp_utc": "2026-08-01T13:58:46.992448Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T135846884628Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T135846884628Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..59db3d1c214b6bb1d3ee637c16809500c0decd33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T135846884628Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c1dbb8a8825040d9160fa5389ad50a9762159f58fc37dd1b1d4ced87f0bcc96 +size 6621490 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T135941162588Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T135941162588Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3bb7bd781e3c95349e9a7d7a8fd3571d0833f073 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T135941162588Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166015488, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 106418180608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166015488, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 61166015488, + "allocated_bytes_peak": 105788905472, + "device": 0, + "inactive_split_bytes_current": 6261615616, + "inactive_split_bytes_peak": 8894116864, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:59:41.162115Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166015488, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105788905472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440304640, + "active_bytes_peak": 106418180608, + "allocated_bytes_current": 72440304640, + "allocated_bytes_peak": 105788905472, + "device": 0, + "inactive_split_bytes_current": 6326627328, + "inactive_split_bytes_peak": 8894116864, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_peak": 112117940224, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:47.007827Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6261615616, + "inactive_split_bytes_current_delta": -65011712, + "inactive_split_bytes_peak": 8894116864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112117940224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112117940224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T135941162588Z.pickle", + "timestamp_utc": "2026-08-01T13:59:41.374343Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T135941162588Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T135941162588Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e93201e47507008d31aa48dae8f97ba927539724 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T135941162588Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9d14973298d11f24257612994ff8e4f0798ebfe8497b4413e28d5a6192a20f9 +size 10534581 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T135837143023Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T135837143023Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0e296e2eef56da690671f8a62f627b0acca0a1a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T135837143023Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 83123062784, + "active_bytes_peak_delta": 47656726528, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 83123062784, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82143936512, + "device": 1, + "inactive_split_bytes_current": 6379310080, + "inactive_split_bytes_peak": 7131797504, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 100736696320, + "reserved_bytes_peak": 100736696320, + "segment_count_current": 469, + "segment_count_peak": 469, + "timestamp_utc": "2026-08-01T13:58:37.142443Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 82143936512, + "allocated_bytes_peak_delta": 46677600256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:57:29.296924Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6379310080, + "inactive_split_bytes_current_delta": 6246491136, + "inactive_split_bytes_peak": 7131797504, + "inactive_split_bytes_peak_delta": 6971411456, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 100736696320, + "reserved_bytes_current_delta": 61492690944, + "reserved_bytes_peak": 100736696320, + "reserved_bytes_peak_delta": 61492690944, + "segment_count_current": 469, + "segment_count_current_delta": 398, + "segment_count_peak": 469, + "segment_count_peak_delta": 398, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T135837143023Z.pickle", + "timestamp_utc": "2026-08-01T13:58:37.260047Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T135837143023Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T135837143023Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5aa846ce180985ac5dfa91f4c70655c2e52e4e12 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T135837143023Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a3996ff97ae8f00b425c288985abde954a7deb6821236480c62bd243c6484b9 +size 5943574 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T135838280863Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T135838280863Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c9efbb56ee5296ab0d80b8b250f3069823b5f5e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T135838280863Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106070848000, + "active_bytes_peak_delta": 22947785216, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106070848000, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 5821467648, + "inactive_split_bytes_peak": 7901402112, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 111459434496, + "reserved_bytes_peak": 111459434496, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:38.280462Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792886272, + "allocated_bytes_peak_delta": 23648949760, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 83123062784, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82143936512, + "device": 1, + "inactive_split_bytes_current": 6379310080, + "inactive_split_bytes_peak": 7131797504, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 100736696320, + "reserved_bytes_peak": 100736696320, + "segment_count_current": 469, + "segment_count_peak": 469, + "timestamp_utc": "2026-08-01T13:58:37.281995Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5821467648, + "inactive_split_bytes_current_delta": -557842432, + "inactive_split_bytes_peak": 7901402112, + "inactive_split_bytes_peak_delta": 769604608, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 111459434496, + "reserved_bytes_current_delta": 10722738176, + "reserved_bytes_peak": 111459434496, + "reserved_bytes_peak_delta": 10722738176, + "segment_count_current": 501, + "segment_count_current_delta": 32, + "segment_count_peak": 501, + "segment_count_peak_delta": 32, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T135838280863Z.pickle", + "timestamp_utc": "2026-08-01T13:58:38.394676Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T135838280863Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T135838280863Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9fc55eb151415dc482ddd11eb0c2405383a950b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T135838280863Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b31991898236288b73bf749d9d0bb8afb3687650210f1711148e9c5902158340 +size 6104340 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T135839419407Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T135839419407Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d433d61a23c89a4961734c18eac0487c002713c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T135839419407Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106139682304, + "active_bytes_peak_delta": 68834304, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106139682304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 6026988544, + "inactive_split_bytes_peak": 8429786112, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 111459434496, + "reserved_bytes_peak": 111459434496, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:39.419012Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792886272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106070848000, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 5821467648, + "inactive_split_bytes_peak": 7901402112, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 111459434496, + "reserved_bytes_peak": 111459434496, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:38.408688Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6026988544, + "inactive_split_bytes_current_delta": 205520896, + "inactive_split_bytes_peak": 8429786112, + "inactive_split_bytes_peak_delta": 528384000, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 111459434496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111459434496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T135839419407Z.pickle", + "timestamp_utc": "2026-08-01T13:58:39.525790Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T135839419407Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T135839419407Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ed444d50e345cec06e83db4b9b70d4795c5d3dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T135839419407Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:777bfc86360e3176764bf0f8f34562ad6f081f66d286708f6041d0d412ab7c49 +size 6621862 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T135840797050Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T135840797050Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8461de2148ab9b68445a50b2d75ff5dae58f8138 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T135840797050Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106139682304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106139682304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 6087805952, + "inactive_split_bytes_peak": 8429786112, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 111459434496, + "reserved_bytes_peak": 111459434496, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:40.796644Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792886272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106139682304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 6026988544, + "inactive_split_bytes_peak": 8429786112, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 111459434496, + "reserved_bytes_peak": 111459434496, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:39.539295Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6087805952, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 8429786112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 111459434496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111459434496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T135840797050Z.pickle", + "timestamp_utc": "2026-08-01T13:58:40.902929Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T135840797050Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T135840797050Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9b5e196fc070a787be1a44461e2b2e6140f36375 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T135840797050Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee1804e9149f33165a952acc077d0bd217109d4545a94f993e00aac8383f76ae +size 6619949 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T135842181507Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T135842181507Z.json new file mode 100644 index 0000000000000000000000000000000000000000..75360a0ecf98982807b2cac84bc1f0ede722c7fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T135842181507Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106139682304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106139682304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 6117166080, + "inactive_split_bytes_peak": 8429786112, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112342335488, + "reserved_bytes_peak": 112342335488, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:42.181109Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792886272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106139682304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 6087805952, + "inactive_split_bytes_peak": 8429786112, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 111459434496, + "reserved_bytes_peak": 111459434496, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:40.916317Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6117166080, + "inactive_split_bytes_current_delta": 29360128, + "inactive_split_bytes_peak": 8429786112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112342335488, + "reserved_bytes_current_delta": 882900992, + "reserved_bytes_peak": 112342335488, + "reserved_bytes_peak_delta": 882900992, + "segment_count_current": 503, + "segment_count_current_delta": 2, + "segment_count_peak": 503, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T135842181507Z.pickle", + "timestamp_utc": "2026-08-01T13:58:42.287139Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T135842181507Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T135842181507Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c97c9e05e158e99fb1de56340cbca5466e1469a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T135842181507Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9af5b461879298c4894e6c6f5e5735af3214516b318eaeec6e0e28f5391a9c4d +size 6097532 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T135843321315Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T135843321315Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2b001829123562747a3dd3eb782095b83526dad3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T135843321315Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106413812224, + "active_bytes_peak_delta": 274129920, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413812224, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 5496409088, + "inactive_split_bytes_peak": 8429786112, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112342335488, + "reserved_bytes_peak": 112342335488, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:43.320897Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792886272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106139682304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 6117166080, + "inactive_split_bytes_peak": 8429786112, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112342335488, + "reserved_bytes_peak": 112342335488, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:42.300519Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5496409088, + "inactive_split_bytes_current_delta": -620756992, + "inactive_split_bytes_peak": 8429786112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112342335488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112342335488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T135843321315Z.pickle", + "timestamp_utc": "2026-08-01T13:58:43.423455Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T135843321315Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T135843321315Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f0208ab734cc86dcd5bb6757ef794624686544bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T135843321315Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9adcdfed5b29257a8120f1d29ce19fd16ccdb253a35bb9983db3fd14b9d577e +size 6619769 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T135844462170Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T135844462170Z.json new file mode 100644 index 0000000000000000000000000000000000000000..85f738489fdcfde6bc16e8134d206303526c9eb2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T135844462170Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106413812224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413812224, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 6469487616, + "inactive_split_bytes_peak": 8429786112, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112342335488, + "reserved_bytes_peak": 112342335488, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:44.461734Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792886272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413812224, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 5496409088, + "inactive_split_bytes_peak": 8429786112, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112342335488, + "reserved_bytes_peak": 112342335488, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:43.436544Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6469487616, + "inactive_split_bytes_current_delta": 973078528, + "inactive_split_bytes_peak": 8429786112, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112342335488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112342335488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T135844462170Z.pickle", + "timestamp_utc": "2026-08-01T13:58:44.565469Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T135844462170Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T135844462170Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dbd1086ebd07226fa2201e959fed77190325a392 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T135844462170Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e1f066edf26b730bf9556fd273e6921be6eb9e87659a122bb39aa048b616f51 +size 6619724 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T135845749074Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T135845749074Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bbe1a93b82fcebcdac99ab420db0e1b1023acabb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T135845749074Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106413812224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413812224, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 5792107520, + "inactive_split_bytes_peak": 8950708224, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112342335488, + "reserved_bytes_peak": 112342335488, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:45.748648Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792886272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413812224, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 6469487616, + "inactive_split_bytes_peak": 8429786112, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112342335488, + "reserved_bytes_peak": 112342335488, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:44.578661Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5792107520, + "inactive_split_bytes_current_delta": -677380096, + "inactive_split_bytes_peak": 8950708224, + "inactive_split_bytes_peak_delta": 520922112, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112342335488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112342335488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T135845749074Z.pickle", + "timestamp_utc": "2026-08-01T13:58:45.855014Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T135845749074Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T135845749074Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e13aaa6ec3fbc3c968ef4898e96ea8df0ec2e992 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T135845749074Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec4fa14e076d6ae613a53dd58cc16ac8e546d02e2ea24fbf91fd403713cbdf3b +size 6619603 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T135846884658Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T135846884658Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7e2ee294457ae92ec1916278a06a82be671fbaa9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T135846884658Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106413812224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413812224, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 5913742336, + "inactive_split_bytes_peak": 8950708224, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112342335488, + "reserved_bytes_peak": 112342335488, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:46.884267Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792886272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413812224, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 5792107520, + "inactive_split_bytes_peak": 8950708224, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112342335488, + "reserved_bytes_peak": 112342335488, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:45.870067Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5913742336, + "inactive_split_bytes_current_delta": 121634816, + "inactive_split_bytes_peak": 8950708224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112342335488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112342335488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T135846884658Z.pickle", + "timestamp_utc": "2026-08-01T13:58:46.990266Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T135846884658Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T135846884658Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5907d95a23fa291c9ad0a390eba68e36b65b2f96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T135846884658Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d8ec5d954747ffa7ea48e8b5ff558af65c61e2625164861a7bbb34d7c96a333 +size 6619708 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T135941162605Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T135941162605Z.json new file mode 100644 index 0000000000000000000000000000000000000000..064625f522f18a1c3eaddb7700edc4a1caaeb3e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T135941162605Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 106413812224, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 106413812224, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 6435933184, + "inactive_split_bytes_peak": 8950708224, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112342335488, + "reserved_bytes_peak": 112342335488, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:59:41.162098Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105792886272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413812224, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105792886272, + "device": 1, + "inactive_split_bytes_current": 5913742336, + "inactive_split_bytes_peak": 8950708224, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112342335488, + "reserved_bytes_peak": 112342335488, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T13:58:47.005609Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6435933184, + "inactive_split_bytes_current_delta": 522190848, + "inactive_split_bytes_peak": 8950708224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112342335488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112342335488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T135941162605Z.pickle", + "timestamp_utc": "2026-08-01T13:59:41.362780Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T135941162605Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T135941162605Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ce86b5bda58e259bd044b626c1e6442e173de3ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T135941162605Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aba9705766e0926ff93d4477ea9decc43a2860565a5b7dad0d9ab0c164403dbb +size 10534579 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T135837142973Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T135837142973Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42ee3d042eadbbd8e44bf381cb7c5359f0783c0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T135837142973Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 83129878528, + "active_bytes_peak_delta": 47663542272, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 83129878528, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82147738624, + "device": 2, + "inactive_split_bytes_current": 6966512640, + "inactive_split_bytes_peak": 7540742144, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 100619255808, + "reserved_bytes_peak": 100619255808, + "segment_count_current": 465, + "segment_count_peak": 465, + "timestamp_utc": "2026-08-01T13:58:37.142407Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 82147738624, + "allocated_bytes_peak_delta": 46681402368, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:57:29.296909Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6966512640, + "inactive_split_bytes_current_delta": 6833693696, + "inactive_split_bytes_peak": 7540742144, + "inactive_split_bytes_peak_delta": 7380356096, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 100619255808, + "reserved_bytes_current_delta": 61375250432, + "reserved_bytes_peak": 100619255808, + "reserved_bytes_peak_delta": 61375250432, + "segment_count_current": 465, + "segment_count_current_delta": 394, + "segment_count_peak": 465, + "segment_count_peak_delta": 394, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T135837142973Z.pickle", + "timestamp_utc": "2026-08-01T13:58:37.257240Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T135837142973Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T135837142973Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..10c0882be40f18cb5c9b0236d3dcfa766953b7cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T135837142973Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f60ddd0ac89d47d5030238d4be0fa4618a1b9eb3f5e689f818ea1700f5bd019 +size 5943013 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T135838280875Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T135838280875Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f1d54351c22f41cbc547e5918d08696ac93a16e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T135838280875Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106134058496, + "active_bytes_peak_delta": 23004179968, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106134058496, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784476672, + "device": 2, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_peak": 7540742144, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 111159541760, + "reserved_bytes_peak": 111159541760, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:58:38.280452Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784476672, + "allocated_bytes_peak_delta": 23636738048, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 83129878528, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82147738624, + "device": 2, + "inactive_split_bytes_current": 6966512640, + "inactive_split_bytes_peak": 7540742144, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 100619255808, + "reserved_bytes_peak": 100619255808, + "segment_count_current": 465, + "segment_count_peak": 465, + "timestamp_utc": "2026-08-01T13:58:37.272505Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_current_delta": -56623104, + "inactive_split_bytes_peak": 7540742144, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 111159541760, + "reserved_bytes_current_delta": 10540285952, + "reserved_bytes_peak": 111159541760, + "reserved_bytes_peak_delta": 10540285952, + "segment_count_current": 497, + "segment_count_current_delta": 32, + "segment_count_peak": 497, + "segment_count_peak_delta": 32, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T135838280875Z.pickle", + "timestamp_utc": "2026-08-01T13:58:38.395581Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T135838280875Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T135838280875Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e5895acadd5a6b7167c41be260c6e99e618abf3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T135838280875Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3f9970a193c4f0fc045039b7596f2c33312580f494f3071f1d6f7d0c5be04cf +size 6625601 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T135839419450Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T135839419450Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e80d703017aec9c3843823087a391cbba8dd872 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T135839419450Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106134058496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106134058496, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784781312, + "device": 2, + "inactive_split_bytes_current": 7469829120, + "inactive_split_bytes_peak": 9200804864, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:39.419038Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784781312, + "allocated_bytes_peak_delta": 304640, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106134058496, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784476672, + "device": 2, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_peak": 7540742144, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 111159541760, + "reserved_bytes_peak": 111159541760, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:58:38.407952Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7469829120, + "inactive_split_bytes_current_delta": 559939584, + "inactive_split_bytes_peak": 9200804864, + "inactive_split_bytes_peak_delta": 1660062720, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112747085824, + "reserved_bytes_current_delta": 1587544064, + "reserved_bytes_peak": 112747085824, + "reserved_bytes_peak_delta": 1587544064, + "segment_count_current": 501, + "segment_count_current_delta": 4, + "segment_count_peak": 501, + "segment_count_peak_delta": 4, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T135839419450Z.pickle", + "timestamp_utc": "2026-08-01T13:58:39.526342Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T135839419450Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T135839419450Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..89433a2c873c35b9d5c7a3ac2f5d45fbfc75dcc8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T135839419450Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a72084ad51ab5a338d08eb639cf5a39cda90a987255da6515a695390ac364159 +size 6086679 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T135840797009Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T135840797009Z.json new file mode 100644 index 0000000000000000000000000000000000000000..44227dbc90009e3ed97f779a3204863954a377dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T135840797009Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106134058496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106134058496, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784781312, + "device": 2, + "inactive_split_bytes_current": 6966512640, + "inactive_split_bytes_peak": 9200804864, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:40.796599Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105784781312, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106134058496, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784781312, + "device": 2, + "inactive_split_bytes_current": 7469829120, + "inactive_split_bytes_peak": 9200804864, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:39.540872Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6966512640, + "inactive_split_bytes_current_delta": -503316480, + "inactive_split_bytes_peak": 9200804864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112747085824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112747085824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T135840797009Z.pickle", + "timestamp_utc": "2026-08-01T13:58:40.903240Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T135840797009Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T135840797009Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cbfaaedf189e2da9ae832a1b6e63180097a65158 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T135840797009Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da01ee6a314221bdd11a96e1db25dd40f29ea462d3bd32e7a186a05660ae50ca +size 6620702 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T135842181485Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T135842181485Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3cff520f0f5c66781f27a30c0804f6914f12f3e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T135842181485Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106134058496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106134058496, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789499904, + "device": 2, + "inactive_split_bytes_current": 7031524352, + "inactive_split_bytes_peak": 9200804864, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:42.181077Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789499904, + "allocated_bytes_peak_delta": 4718592, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106134058496, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105784781312, + "device": 2, + "inactive_split_bytes_current": 6966512640, + "inactive_split_bytes_peak": 9200804864, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:40.915316Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7031524352, + "inactive_split_bytes_current_delta": 65011712, + "inactive_split_bytes_peak": 9200804864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112747085824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112747085824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T135842181485Z.pickle", + "timestamp_utc": "2026-08-01T13:58:42.291586Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T135842181485Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T135842181485Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5c80ccd3f54595dbf49d9a7c064f869f4a06a157 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T135842181485Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecebde7f4501fa1ab3777afd08ffa96ad1e919f0b5ea458db96481fe88f9f2d3 +size 6620368 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T135843321245Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T135843321245Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a64eeb54cde21092a1de68e3bdb8e27a4692144 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T135843321245Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106413673984, + "active_bytes_peak_delta": 279615488, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413673984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789499904, + "device": 2, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_peak": 9200804864, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:43.320850Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789499904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106134058496, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789499904, + "device": 2, + "inactive_split_bytes_current": 7031524352, + "inactive_split_bytes_peak": 9200804864, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:42.303649Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_current_delta": -121634816, + "inactive_split_bytes_peak": 9200804864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112747085824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112747085824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T135843321245Z.pickle", + "timestamp_utc": "2026-08-01T13:58:43.419300Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T135843321245Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T135843321245Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f4bef6b7cc22c20872011c35d7a9425e0c150c7f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T135843321245Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:194555920542428464cbc129b5ee38e2acb8f56307ad3853aadc4e89ae5be9f3 +size 6620268 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T135844462068Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T135844462068Z.json new file mode 100644 index 0000000000000000000000000000000000000000..372bc98f5545ca9b0a78352a1f0a1b8965b93bc0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T135844462068Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106413673984, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413673984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789499904, + "device": 2, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_peak": 9200804864, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:44.461674Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789499904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413673984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789499904, + "device": 2, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_peak": 9200804864, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:43.430763Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9200804864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112747085824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112747085824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T135844462068Z.pickle", + "timestamp_utc": "2026-08-01T13:58:44.561078Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T135844462068Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T135844462068Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94072d5e7eac972756cf8aa0362e9d925fdadeaa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T135844462068Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa77ac7c8ce603626860752f859d00e81308533b3e22b6a5eb994c8ad883f763 +size 6620143 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T135845749006Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T135845749006Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d737e99c6a99faa3a5be2bda957e7ba7b01492b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T135845749006Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106413673984, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413673984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789499904, + "device": 2, + "inactive_split_bytes_current": 6496750592, + "inactive_split_bytes_peak": 9200804864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:45.748597Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789499904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413673984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789499904, + "device": 2, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_peak": 9200804864, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:44.572648Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6496750592, + "inactive_split_bytes_current_delta": -413138944, + "inactive_split_bytes_peak": 9200804864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112747085824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112747085824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T135845749006Z.pickle", + "timestamp_utc": "2026-08-01T13:58:45.845627Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T135845749006Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T135845749006Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..736caea47a232739f9c48b76e91e76f68656c55f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T135845749006Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bc66ec139e6e844db759092925e9dfa50e84c052a59ae3d1aed8facf17906ca +size 6619915 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T135846884664Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T135846884664Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4b3e869ea4e6d88b7b589520d9bc0fc3885aeff6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T135846884664Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106413673984, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413673984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789499904, + "device": 2, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_peak": 9200804864, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:46.884250Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789499904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413673984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789499904, + "device": 2, + "inactive_split_bytes_current": 6496750592, + "inactive_split_bytes_peak": 9200804864, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:45.856890Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_current_delta": 413138944, + "inactive_split_bytes_peak": 9200804864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112747085824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112747085824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T135846884664Z.pickle", + "timestamp_utc": "2026-08-01T13:58:46.983616Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T135846884664Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T135846884664Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..503712cb53f3960c91ce46fead73cb6bc3186726 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T135846884664Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5418eeff96e83d7654f958df315d4fd4ff48aeb97f44fa1903d166583b1d16f9 +size 6619985 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T135941162672Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T135941162672Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65f951ebbf0bdea91c54f3dd0104e86a1f5b2c7a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T135941162672Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 106413673984, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 106413673984, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 105789499904, + "device": 2, + "inactive_split_bytes_current": 6379310080, + "inactive_split_bytes_peak": 9200804864, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:59:41.162115Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105789499904, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106413673984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789499904, + "device": 2, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_peak": 9200804864, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112747085824, + "reserved_bytes_peak": 112747085824, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T13:58:46.994941Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6379310080, + "inactive_split_bytes_current_delta": -530579456, + "inactive_split_bytes_peak": 9200804864, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112747085824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112747085824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T135941162672Z.pickle", + "timestamp_utc": "2026-08-01T13:59:41.372186Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T135941162672Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T135941162672Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e3378a36dd515ef47ab4b59a2cb95aa8bd74b99e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T135941162672Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bac9c41dd3b6daa364d69ff0e49da2f5cfa107218d6899fccd2877e57177df34 +size 10534977 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T135837143021Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T135837143021Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4407c96a5124bace17ce69ad640443d8973be860 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T135837143021Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 83132499968, + "active_bytes_peak_delta": 47666163712, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 83132499968, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82149311488, + "device": 3, + "inactive_split_bytes_current": 6966512640, + "inactive_split_bytes_peak": 6966513152, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 100501815296, + "reserved_bytes_peak": 100501815296, + "segment_count_current": 465, + "segment_count_peak": 465, + "timestamp_utc": "2026-08-01T13:58:37.142422Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 82149311488, + "allocated_bytes_peak_delta": 46682975232, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:57:29.296918Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6966512640, + "inactive_split_bytes_current_delta": 6833693696, + "inactive_split_bytes_peak": 6966513152, + "inactive_split_bytes_peak_delta": 6806127104, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 100501815296, + "reserved_bytes_current_delta": 61257809920, + "reserved_bytes_peak": 100501815296, + "reserved_bytes_peak_delta": 61257809920, + "segment_count_current": 465, + "segment_count_current_delta": 394, + "segment_count_peak": 465, + "segment_count_peak_delta": 394, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T135837143021Z.pickle", + "timestamp_utc": "2026-08-01T13:58:37.260515Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T135837143021Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T135837143021Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b92bdc9607e7412595aa2220c6903a721792acdc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T135837143021Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7bcd02cb39d0fea5434330ed120e35d68cdfe525ccecfb699a1c7b2414a294a +size 5942984 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T135838280939Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T135838280939Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a5f5084cf7686d752743aa0a62e95de696540662 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T135838280939Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106416041472, + "active_bytes_peak_delta": 23283541504, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105786223104, + "device": 3, + "inactive_split_bytes_current": 6379310080, + "inactive_split_bytes_peak": 8407318528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110874329088, + "reserved_bytes_peak": 110874329088, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:38.280482Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105786223104, + "allocated_bytes_peak_delta": 23636911616, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 83132499968, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82149311488, + "device": 3, + "inactive_split_bytes_current": 6966512640, + "inactive_split_bytes_peak": 6966513152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 100501815296, + "reserved_bytes_peak": 100501815296, + "segment_count_current": 465, + "segment_count_peak": 465, + "timestamp_utc": "2026-08-01T13:58:37.277397Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6379310080, + "inactive_split_bytes_current_delta": -587202560, + "inactive_split_bytes_peak": 8407318528, + "inactive_split_bytes_peak_delta": 1440805376, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110874329088, + "reserved_bytes_current_delta": 10372513792, + "reserved_bytes_peak": 110874329088, + "reserved_bytes_peak_delta": 10372513792, + "segment_count_current": 495, + "segment_count_current_delta": 30, + "segment_count_peak": 495, + "segment_count_peak_delta": 30, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T135838280939Z.pickle", + "timestamp_utc": "2026-08-01T13:58:38.396692Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T135838280939Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T135838280939Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb712cd8979bfe826f054f069b0343bee50bc509 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T135838280939Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64548419148d3bb5abda68261ec868cfccacffcb8bd174c5a8731f9c53b6c9cd +size 6102501 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T135839419415Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T135839419415Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fc9d645aa96eb32b328f04e951dd9457588ceda7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T135839419415Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106416041472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 7056690176, + "inactive_split_bytes_peak": 8407318528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110874329088, + "reserved_bytes_peak": 110874329088, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:39.419006Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789216256, + "allocated_bytes_peak_delta": 2993152, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105786223104, + "device": 3, + "inactive_split_bytes_current": 6379310080, + "inactive_split_bytes_peak": 8407318528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110874329088, + "reserved_bytes_peak": 110874329088, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:38.410831Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7056690176, + "inactive_split_bytes_current_delta": 677380096, + "inactive_split_bytes_peak": 8407318528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110874329088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110874329088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 495, + "segment_count_current_delta": 0, + "segment_count_peak": 495, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T135839419415Z.pickle", + "timestamp_utc": "2026-08-01T13:58:39.526756Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T135839419415Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T135839419415Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d849923bce8a8678b0197259a7bff61e2acc24f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T135839419415Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bbf02b927138e5bf5e47d8688fd55decc52481fc34f516b3ce4d652aa0fde83 +size 6341580 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T135840797018Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T135840797018Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c298da94e64d04821ad031eeedc5c1de332b1e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T135840797018Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106416041472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 6496750592, + "inactive_split_bytes_peak": 8407318528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110874329088, + "reserved_bytes_peak": 110874329088, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:40.796595Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789216256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 7056690176, + "inactive_split_bytes_peak": 8407318528, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110874329088, + "reserved_bytes_peak": 110874329088, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:39.540599Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6496750592, + "inactive_split_bytes_current_delta": -559939584, + "inactive_split_bytes_peak": 8407318528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110874329088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110874329088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 495, + "segment_count_current_delta": 0, + "segment_count_peak": 495, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T135840797018Z.pickle", + "timestamp_utc": "2026-08-01T13:58:40.905816Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T135840797018Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T135840797018Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e909f99bcbe779d355da1b3bf04a48b24d010387 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T135840797018Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3acc3028e1f47656f3d5c6f6059955aed7bb41cfe8ba50554819c6e03937511 +size 6341458 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T135842181581Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T135842181581Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9fcc7933d7ef479c323e955a5949fecd908fb5b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T135842181581Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106416041472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 6765186048, + "inactive_split_bytes_peak": 8407318528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110874329088, + "reserved_bytes_peak": 110874329088, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:42.181183Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789216256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 6496750592, + "inactive_split_bytes_peak": 8407318528, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110874329088, + "reserved_bytes_peak": 110874329088, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:40.919716Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6765186048, + "inactive_split_bytes_current_delta": 268435456, + "inactive_split_bytes_peak": 8407318528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110874329088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110874329088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 495, + "segment_count_current_delta": 0, + "segment_count_peak": 495, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T135842181581Z.pickle", + "timestamp_utc": "2026-08-01T13:58:42.289694Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T135842181581Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T135842181581Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..253c32d3455dc9295fc0a81900349541e52a50a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T135842181581Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92216a71d7127da8ec8c541712e415f693c567d7c094a57081440a3f75a02a4b +size 6613887 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T135843321305Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T135843321305Z.json new file mode 100644 index 0000000000000000000000000000000000000000..02d221520aef0cad5792533e23f34baa33fc67cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T135843321305Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106416041472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 6651939840, + "inactive_split_bytes_peak": 8407318528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110874329088, + "reserved_bytes_peak": 110874329088, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:43.320901Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789216256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 6765186048, + "inactive_split_bytes_peak": 8407318528, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110874329088, + "reserved_bytes_peak": 110874329088, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:42.303381Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6651939840, + "inactive_split_bytes_current_delta": -113246208, + "inactive_split_bytes_peak": 8407318528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110874329088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110874329088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 495, + "segment_count_current_delta": 0, + "segment_count_peak": 495, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T135843321305Z.pickle", + "timestamp_utc": "2026-08-01T13:58:43.424222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T135843321305Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T135843321305Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4abd47a52997fbed7217fdd34c0e0c45a7f9f950 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T135843321305Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f29d088c4dbc903fd97945d1b8f95cf2162da324b23d1ff63a27266904db5f6 +size 6613960 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T135844462121Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T135844462121Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1dcb191950e3807f3c16e1e358e7734ace767ba5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T135844462121Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106416041472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 7060884480, + "inactive_split_bytes_peak": 8407318528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 110874329088, + "reserved_bytes_peak": 110874329088, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:44.461720Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789216256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 6651939840, + "inactive_split_bytes_peak": 8407318528, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110874329088, + "reserved_bytes_peak": 110874329088, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:43.437472Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7060884480, + "inactive_split_bytes_current_delta": 408944640, + "inactive_split_bytes_peak": 8407318528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 110874329088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110874329088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 495, + "segment_count_current_delta": 0, + "segment_count_peak": 495, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T135844462121Z.pickle", + "timestamp_utc": "2026-08-01T13:58:44.574139Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T135844462121Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T135844462121Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f1775ce546f54581fdced1e1310a3c875c88cc7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T135844462121Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2539fcbb1737fa31eab1025b4e934151e9218cf51ad6803fe39a7041e2322fdf +size 6607570 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T135845749036Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T135845749036Z.json new file mode 100644 index 0000000000000000000000000000000000000000..57903b83ca6962772eb04ae4e8d50faa75b46fd6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T135845749036Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106416041472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 7474023424, + "inactive_split_bytes_peak": 8407318528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 111287468032, + "reserved_bytes_peak": 111287468032, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:45.748636Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789216256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 7060884480, + "inactive_split_bytes_peak": 8407318528, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 110874329088, + "reserved_bytes_peak": 110874329088, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:44.590153Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7474023424, + "inactive_split_bytes_current_delta": 413138944, + "inactive_split_bytes_peak": 8407318528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 111287468032, + "reserved_bytes_current_delta": 413138944, + "reserved_bytes_peak": 111287468032, + "reserved_bytes_peak_delta": 413138944, + "segment_count_current": 496, + "segment_count_current_delta": 1, + "segment_count_peak": 496, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T135845749036Z.pickle", + "timestamp_utc": "2026-08-01T13:58:45.851124Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T135845749036Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T135845749036Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..add7f1ef9d25c533a9ee2876d06e043c4d624a3f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T135845749036Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37459ef2cdc204883d4121d4cf0c3a40374c21f61fa6dea369e38ca9a92bc70b +size 6086086 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T135846884675Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T135846884675Z.json new file mode 100644 index 0000000000000000000000000000000000000000..830ae05f5e83977c3bea80e3044524e22ed2d3d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T135846884675Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106416041472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 6914083840, + "inactive_split_bytes_peak": 8407318528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 111287468032, + "reserved_bytes_peak": 111287468032, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:46.884272Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789216256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 7474023424, + "inactive_split_bytes_peak": 8407318528, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 111287468032, + "reserved_bytes_peak": 111287468032, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:45.864471Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6914083840, + "inactive_split_bytes_current_delta": -559939584, + "inactive_split_bytes_peak": 8407318528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 111287468032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111287468032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 496, + "segment_count_current_delta": 0, + "segment_count_peak": 496, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T135846884675Z.pickle", + "timestamp_utc": "2026-08-01T13:58:46.985831Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T135846884675Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T135846884675Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c1840bd5bfe838a4891882a4cfc7f31b8b40370f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T135846884675Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:724875788f323bbd1afdab22c70bf1f374de864b376025f30a258b9a81659bf2 +size 6620183 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T135941162704Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T135941162704Z.json new file mode 100644 index 0000000000000000000000000000000000000000..07ec1294ea27c3cf736bc764994833d3ddcbe43e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T135941162704Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 106416041472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 6731631616, + "inactive_split_bytes_peak": 8407318528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 111287468032, + "reserved_bytes_peak": 111287468032, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:59:41.162113Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105789216256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106416041472, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105789216256, + "device": 3, + "inactive_split_bytes_current": 6914083840, + "inactive_split_bytes_peak": 8407318528, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 111287468032, + "reserved_bytes_peak": 111287468032, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:46.998877Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6731631616, + "inactive_split_bytes_current_delta": -182452224, + "inactive_split_bytes_peak": 8407318528, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 111287468032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111287468032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 496, + "segment_count_current_delta": 0, + "segment_count_peak": 496, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T135941162704Z.pickle", + "timestamp_utc": "2026-08-01T13:59:41.362649Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T135941162704Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T135941162704Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..551a37800cdbc8aae4536dcfa8580c0b34fc6b6a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T135941162704Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86ee72b663178544d12c536bd1c187a130e41decf781ef8a2fa2d980a2855fce +size 9714513 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T135837143012Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T135837143012Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8c76201ea8cd06a16a6255f553b356b8428c97bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T135837143012Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442152960, + "active_bytes_current_delta": 37210255360, + "active_bytes_peak": 83129915392, + "active_bytes_peak_delta": 47663136768, + "after": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 83129915392, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 82147775488, + "device": 4, + "inactive_split_bytes_current": 6463191040, + "inactive_split_bytes_peak": 7380471296, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 101097406464, + "reserved_bytes_peak": 101097406464, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T13:58:37.142425Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442152960, + "allocated_bytes_current_delta": 37210255360, + "allocated_bytes_peak": 82147775488, + "allocated_bytes_peak_delta": 46680996864, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:57:29.296544Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6463191040, + "inactive_split_bytes_current_delta": 6330814464, + "inactive_split_bytes_peak": 7380471296, + "inactive_split_bytes_peak_delta": 7227061760, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 101097406464, + "reserved_bytes_current_delta": 61853401088, + "reserved_bytes_peak": 101097406464, + "reserved_bytes_peak_delta": 61853401088, + "segment_count_current": 466, + "segment_count_current_delta": 395, + "segment_count_peak": 466, + "segment_count_peak_delta": 395, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T135837143012Z.pickle", + "timestamp_utc": "2026-08-01T13:58:37.258945Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T135837143012Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T135837143012Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b3eeb9d0ecb181ce74b24cb93618462ca49b339 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T135837143012Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30a0e672a25baf3618b99df7737f4105797c27e714d6e497df1b446a8820fcb1 +size 5951712 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T135838280834Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T135838280834Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d3086fc1de281e7c7cf76a9cd8f49afbacbfb2d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T135838280834Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442152960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106771466752, + "active_bytes_peak_delta": 23641551360, + "after": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105788409344, + "device": 4, + "inactive_split_bytes_current": 5932611584, + "inactive_split_bytes_peak": 8782837760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 111027421184, + "reserved_bytes_peak": 111027421184, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:38.280432Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442152960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788409344, + "allocated_bytes_peak_delta": 23640633856, + "before": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 83129915392, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 82147775488, + "device": 4, + "inactive_split_bytes_current": 6463191040, + "inactive_split_bytes_peak": 7380471296, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 101097406464, + "reserved_bytes_peak": 101097406464, + "segment_count_current": 466, + "segment_count_peak": 466, + "timestamp_utc": "2026-08-01T13:58:37.277409Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5932611584, + "inactive_split_bytes_current_delta": -530579456, + "inactive_split_bytes_peak": 8782837760, + "inactive_split_bytes_peak_delta": 1402366464, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 111027421184, + "reserved_bytes_current_delta": 9930014720, + "reserved_bytes_peak": 111027421184, + "reserved_bytes_peak_delta": 9930014720, + "segment_count_current": 492, + "segment_count_current_delta": 26, + "segment_count_peak": 492, + "segment_count_peak_delta": 26, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T135838280834Z.pickle", + "timestamp_utc": "2026-08-01T13:58:38.397350Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T135838280834Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T135838280834Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ee3101dd1d4604ebf391416efde088d80467232 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T135838280834Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30202a6a17a2de941b9f06c6f1e2d4d7d54e3e44002c4d2e92f696eeaf2547db +size 6628229 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T135839419396Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T135839419396Z.json new file mode 100644 index 0000000000000000000000000000000000000000..17bbc9a1b0f537488917ab44a9e0d27458843bbf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T135839419396Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442152960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106771466752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105788409344, + "device": 4, + "inactive_split_bytes_current": 6291224576, + "inactive_split_bytes_peak": 8782837760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 111027421184, + "reserved_bytes_peak": 111027421184, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:39.418970Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442152960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788409344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105788409344, + "device": 4, + "inactive_split_bytes_current": 5932611584, + "inactive_split_bytes_peak": 8782837760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 111027421184, + "reserved_bytes_peak": 111027421184, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:38.411458Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6291224576, + "inactive_split_bytes_current_delta": 358612992, + "inactive_split_bytes_peak": 8782837760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 111027421184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111027421184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 492, + "segment_count_current_delta": 0, + "segment_count_peak": 492, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T135839419396Z.pickle", + "timestamp_utc": "2026-08-01T13:58:39.530788Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T135839419396Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T135839419396Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..51a0dbd7b9488ca616ea7e4ed3c35b3cc22ff00a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T135839419396Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf9ef1a745933d7eb33378c25174bb62442e007d43f4ee9d63a7dcb63a98bc23 +size 6621907 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T135840796949Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T135840796949Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3a9b80e4dbc0c71a09c3e67fefa67f54fb508cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T135840796949Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442152960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106771466752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105788409344, + "device": 4, + "inactive_split_bytes_current": 6115063808, + "inactive_split_bytes_peak": 8782837760, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 111027421184, + "reserved_bytes_peak": 111027421184, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:40.796547Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442152960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788409344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105788409344, + "device": 4, + "inactive_split_bytes_current": 6291224576, + "inactive_split_bytes_peak": 8782837760, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 111027421184, + "reserved_bytes_peak": 111027421184, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:39.544655Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6115063808, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 8782837760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 111027421184, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111027421184, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 492, + "segment_count_current_delta": 0, + "segment_count_peak": 492, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T135840796949Z.pickle", + "timestamp_utc": "2026-08-01T13:58:40.906498Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T135840796949Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T135840796949Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd9b2b16fa0e2115a0cfb0ac54a45a2b37c9c13d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T135840796949Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59f40c6f3f770a2719f6f546d2a486552f57b2cfcd9c506638051d9fe850d13b +size 6620825 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T135842181490Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T135842181490Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f90882ba05c1570884c06796885bc22c3dceac4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T135842181490Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442152960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106771466752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105788409344, + "device": 4, + "inactive_split_bytes_current": 6440122368, + "inactive_split_bytes_peak": 8897591296, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112262643712, + "reserved_bytes_peak": 112262643712, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:42.181065Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442152960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788409344, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105788409344, + "device": 4, + "inactive_split_bytes_current": 6115063808, + "inactive_split_bytes_peak": 8782837760, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 111027421184, + "reserved_bytes_peak": 111027421184, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:40.920114Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6440122368, + "inactive_split_bytes_current_delta": 325058560, + "inactive_split_bytes_peak": 8897591296, + "inactive_split_bytes_peak_delta": 114753536, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112262643712, + "reserved_bytes_current_delta": 1235222528, + "reserved_bytes_peak": 112262643712, + "reserved_bytes_peak_delta": 1235222528, + "segment_count_current": 495, + "segment_count_current_delta": 3, + "segment_count_peak": 495, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T135842181490Z.pickle", + "timestamp_utc": "2026-08-01T13:58:42.289914Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T135842181490Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T135842181490Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4aba13c05ca885d348d9c849e77702d7b7f05172 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T135842181490Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03f25b808c4ad9b94ad0a8e8018f90f67fde35c4c3b6175075e6e9d32e7fc2bb +size 6624018 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T135843321249Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T135843321249Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c1a9720cdc2e56c7177e5c3bfce53e0ae0c3a6f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T135843321249Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442152960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106771466752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105789326848, + "device": 4, + "inactive_split_bytes_current": 5641107456, + "inactive_split_bytes_peak": 8897591296, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112262643712, + "reserved_bytes_peak": 112262643712, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:43.320841Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442152960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789326848, + "allocated_bytes_peak_delta": 917504, + "before": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105788409344, + "device": 4, + "inactive_split_bytes_current": 6440122368, + "inactive_split_bytes_peak": 8897591296, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112262643712, + "reserved_bytes_peak": 112262643712, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:42.303523Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5641107456, + "inactive_split_bytes_current_delta": -799014912, + "inactive_split_bytes_peak": 8897591296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112262643712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112262643712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 495, + "segment_count_current_delta": 0, + "segment_count_peak": 495, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T135843321249Z.pickle", + "timestamp_utc": "2026-08-01T13:58:43.425940Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T135843321249Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T135843321249Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..73bd317d2813e064afb12b68e7faa0a730142d2a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T135843321249Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5483a1f117cc3610cc3e6eea655a9fec565c33f20cf1c9fa364cf52a11f5c42f +size 6622198 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T135844462093Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T135844462093Z.json new file mode 100644 index 0000000000000000000000000000000000000000..078d62cffda2e1fb7e3116115ed45a8b5a5a8e7e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T135844462093Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442152960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106771466752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105789326848, + "device": 4, + "inactive_split_bytes_current": 6580631552, + "inactive_split_bytes_peak": 8897591296, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112262643712, + "reserved_bytes_peak": 112262643712, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:44.461677Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442152960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789326848, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105789326848, + "device": 4, + "inactive_split_bytes_current": 5641107456, + "inactive_split_bytes_peak": 8897591296, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112262643712, + "reserved_bytes_peak": 112262643712, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:43.439375Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6580631552, + "inactive_split_bytes_current_delta": 939524096, + "inactive_split_bytes_peak": 8897591296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112262643712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112262643712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 495, + "segment_count_current_delta": 0, + "segment_count_peak": 495, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T135844462093Z.pickle", + "timestamp_utc": "2026-08-01T13:58:44.574257Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T135844462093Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T135844462093Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f9ccfce26c1d764ac968ecd056bb1d27d5625ceb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T135844462093Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44ced1730a0fbeddc8617822e7548aecac090784c8fb09a272c59405a4e7db60 +size 6637671 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T135845748981Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T135845748981Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f023fb1318b9a08a4659e915687676859b07fc6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T135845748981Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442152960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106771466752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105789326848, + "device": 4, + "inactive_split_bytes_current": 6524008448, + "inactive_split_bytes_peak": 8897591296, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112262643712, + "reserved_bytes_peak": 112262643712, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:45.748590Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442152960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789326848, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105789326848, + "device": 4, + "inactive_split_bytes_current": 6580631552, + "inactive_split_bytes_peak": 8897591296, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112262643712, + "reserved_bytes_peak": 112262643712, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:44.590253Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6524008448, + "inactive_split_bytes_current_delta": -56623104, + "inactive_split_bytes_peak": 8897591296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112262643712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112262643712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 495, + "segment_count_current_delta": 0, + "segment_count_peak": 495, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T135845748981Z.pickle", + "timestamp_utc": "2026-08-01T13:58:45.853673Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T135845748981Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T135845748981Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..63f0b25db02ef09f15741963bacbf041f59a21eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T135845748981Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74e87d5c39fe0f138d168dcec1d979d0c6a8e315bc66c1817cb8a4ab7471e952 +size 6620041 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T135846884633Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T135846884633Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c91dbf54643112497d8597dc58f9a15852790a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T135846884633Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442152960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106771466752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105789326848, + "device": 4, + "inactive_split_bytes_current": 6467385344, + "inactive_split_bytes_peak": 8897591296, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112262643712, + "reserved_bytes_peak": 112262643712, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:46.884233Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442152960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789326848, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105789326848, + "device": 4, + "inactive_split_bytes_current": 6524008448, + "inactive_split_bytes_peak": 8897591296, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112262643712, + "reserved_bytes_peak": 112262643712, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:45.867293Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6467385344, + "inactive_split_bytes_current_delta": -56623104, + "inactive_split_bytes_peak": 8897591296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112262643712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112262643712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 495, + "segment_count_current_delta": 0, + "segment_count_peak": 495, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T135846884633Z.pickle", + "timestamp_utc": "2026-08-01T13:58:46.991352Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T135846884633Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T135846884633Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8555b8814b2db63d2d9fb7d717c31d6b14f5a73f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T135846884633Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:952a3517588b6de81e708d7694d75c5aa409a10c0ab605fc9e12a55a708c69b6 +size 6618736 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T135941162602Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T135941162602Z.json new file mode 100644 index 0000000000000000000000000000000000000000..78ab54b879a6e90c2e91702c5005987ebf80eb0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T135941162602Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61167863808, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 106771466752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61167863808, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 61167863808, + "allocated_bytes_peak": 105789326848, + "device": 4, + "inactive_split_bytes_current": 6050052096, + "inactive_split_bytes_peak": 8897591296, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112262643712, + "reserved_bytes_peak": 112262643712, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:59:41.162111Z", + "world_size": 8 + }, + "allocated_bytes_current": 61167863808, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105789326848, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442152960, + "active_bytes_peak": 106771466752, + "allocated_bytes_current": 72442152960, + "allocated_bytes_peak": 105789326848, + "device": 4, + "inactive_split_bytes_current": 6467385344, + "inactive_split_bytes_peak": 8897591296, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112262643712, + "reserved_bytes_peak": 112262643712, + "segment_count_current": 495, + "segment_count_peak": 495, + "timestamp_utc": "2026-08-01T13:58:47.004935Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6050052096, + "inactive_split_bytes_current_delta": -417333248, + "inactive_split_bytes_peak": 8897591296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112262643712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112262643712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 495, + "segment_count_current_delta": 0, + "segment_count_peak": 495, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T135941162602Z.pickle", + "timestamp_utc": "2026-08-01T13:59:41.363715Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T135941162602Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T135941162602Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ff722fe4de5f3e0ae9827f40b7caeffb74f2fb50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T135941162602Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bca8b678a926bae1d7c8f180183b6bd8ba48d40a3d634efb2bfab745a5a566dc +size 10533037 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T135837143137Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T135837143137Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d0cd5534ab00fd4f8fa1020e20979fa80a6562d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T135837143137Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 83118446592, + "active_bytes_peak_delta": 47652110336, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 83118446592, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82146426880, + "device": 5, + "inactive_split_bytes_current": 7144770560, + "inactive_split_bytes_peak": 7544936448, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 100738793472, + "reserved_bytes_peak": 100738793472, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T13:58:37.142491Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 82146426880, + "allocated_bytes_peak_delta": 46680090624, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:57:29.296824Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7144770560, + "inactive_split_bytes_current_delta": 7011951616, + "inactive_split_bytes_peak": 7544936448, + "inactive_split_bytes_peak_delta": 7384550400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 100738793472, + "reserved_bytes_current_delta": 61494788096, + "reserved_bytes_peak": 100738793472, + "reserved_bytes_peak_delta": 61494788096, + "segment_count_current": 463, + "segment_count_current_delta": 392, + "segment_count_peak": 463, + "segment_count_peak_delta": 392, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T135837143137Z.pickle", + "timestamp_utc": "2026-08-01T13:58:37.260192Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T135837143137Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T135837143137Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76f3c9924db6830a63a0a70d5575680f5565c239 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T135837143137Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f5a277a647f774506c8b2611826b07ab63bc64834a030abf476ca3b0306bba3 +size 5942851 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T135838282498Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T135838282498Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d8e704d5a3213f9ec14cc95975c1041c0967a72 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T135838282498Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418379264, + "active_bytes_peak_delta": 23299932672, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787753472, + "device": 5, + "inactive_split_bytes_current": 6849072128, + "inactive_split_bytes_peak": 9434158080, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 111490891776, + "reserved_bytes_peak": 111490891776, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:58:38.282069Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787753472, + "allocated_bytes_peak_delta": 23641326592, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 83118446592, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82146426880, + "device": 5, + "inactive_split_bytes_current": 7144770560, + "inactive_split_bytes_peak": 7544936448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 100738793472, + "reserved_bytes_peak": 100738793472, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T13:58:37.279107Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6849072128, + "inactive_split_bytes_current_delta": -295698432, + "inactive_split_bytes_peak": 9434158080, + "inactive_split_bytes_peak_delta": 1889221632, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 111490891776, + "reserved_bytes_current_delta": 10752098304, + "reserved_bytes_peak": 111490891776, + "reserved_bytes_peak_delta": 10752098304, + "segment_count_current": 491, + "segment_count_current_delta": 28, + "segment_count_peak": 491, + "segment_count_peak_delta": 28, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T135838282498Z.pickle", + "timestamp_utc": "2026-08-01T13:58:38.402071Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T135838282498Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T135838282498Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aab430452309ac89ad1cc0537ec3ff6a5ccffb36 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T135838282498Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ffc1d7957a115b7af6cfb1085ff326fbe6f55c1465d1a03d33138b5b4b0b93a +size 6100577 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T135839419420Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T135839419420Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c3e87402bd4c90f41e3c8b9543766e13f3304ee5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T135839419420Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418379264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 6735825920, + "inactive_split_bytes_peak": 9434158080, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 111490891776, + "reserved_bytes_peak": 111490891776, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:58:39.419013Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105791051264, + "allocated_bytes_peak_delta": 3297792, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105787753472, + "device": 5, + "inactive_split_bytes_current": 6849072128, + "inactive_split_bytes_peak": 9434158080, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 111490891776, + "reserved_bytes_peak": 111490891776, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:58:38.418106Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6735825920, + "inactive_split_bytes_current_delta": -113246208, + "inactive_split_bytes_peak": 9434158080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 111490891776, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111490891776, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 491, + "segment_count_current_delta": 0, + "segment_count_peak": 491, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T135839419420Z.pickle", + "timestamp_utc": "2026-08-01T13:58:39.537982Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T135839419420Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T135839419420Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf4ae061072174eb646b43513c170098aa904c0c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T135839419420Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3eb99d977581a2eba3a48dd1e796cad3807983268c65875c98a88b405c053efe +size 6620639 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T135840797005Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T135840797005Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8fbc6c90da489c483bc8ee86f426a6f23147aafc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T135840797005Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418379264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 7148964864, + "inactive_split_bytes_peak": 9434158080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 111490891776, + "reserved_bytes_peak": 111490891776, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:58:40.796590Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105791051264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 6735825920, + "inactive_split_bytes_peak": 9434158080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 111490891776, + "reserved_bytes_peak": 111490891776, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:58:39.556416Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7148964864, + "inactive_split_bytes_current_delta": 413138944, + "inactive_split_bytes_peak": 9434158080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 111490891776, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111490891776, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 491, + "segment_count_current_delta": 0, + "segment_count_peak": 491, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T135840797005Z.pickle", + "timestamp_utc": "2026-08-01T13:58:40.909491Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T135840797005Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T135840797005Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3246227ee6297cefc39ecb581c3898e0fd589af4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T135840797005Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23a2d0645e0a18fa30cbc9ca4769f27349548e999735a9265cff1656098793a1 +size 6619487 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T135842181595Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T135842181595Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9274f0bafdd467525b3b24385a16b8179ec85a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T135842181595Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418379264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 7614532608, + "inactive_split_bytes_peak": 9434158080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 111490891776, + "reserved_bytes_peak": 111490891776, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:58:42.181183Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105791051264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 7148964864, + "inactive_split_bytes_peak": 9434158080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 111490891776, + "reserved_bytes_peak": 111490891776, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:58:40.925193Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7614532608, + "inactive_split_bytes_current_delta": 465567744, + "inactive_split_bytes_peak": 9434158080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 111490891776, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111490891776, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 491, + "segment_count_current_delta": 0, + "segment_count_peak": 491, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T135842181595Z.pickle", + "timestamp_utc": "2026-08-01T13:58:42.293523Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T135842181595Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T135842181595Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dce4e4af66184980d3bef72e6c0ae8845db19e91 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T135842181595Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5b2989599ce370abb89fe58eb820ab24ea520bfa5661c99bbf57c308e187636 +size 6619310 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T135843321291Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T135843321291Z.json new file mode 100644 index 0000000000000000000000000000000000000000..41a043570cb76cdc932641fb8a6ee1ad26b702b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T135843321291Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418379264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 6675008512, + "inactive_split_bytes_peak": 9434158080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 111490891776, + "reserved_bytes_peak": 111490891776, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:58:43.320889Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105791051264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 7614532608, + "inactive_split_bytes_peak": 9434158080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 111490891776, + "reserved_bytes_peak": 111490891776, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:58:42.309022Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6675008512, + "inactive_split_bytes_current_delta": -939524096, + "inactive_split_bytes_peak": 9434158080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 111490891776, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111490891776, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 491, + "segment_count_current_delta": 0, + "segment_count_peak": 491, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T135843321291Z.pickle", + "timestamp_utc": "2026-08-01T13:58:43.430250Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T135843321291Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T135843321291Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6a754598e4d75848f71708b27985812ae3dfda2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T135843321291Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1cd9ae4b377854d5af7926237fa1399fc11f8833e939f7f5165f3ed4ee45a8b +size 6618967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T135844462115Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T135844462115Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9cdc3e3b4360aed85db0ecea78b592f47ee30321 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T135844462115Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418379264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 8145112064, + "inactive_split_bytes_peak": 9434158080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 111904030720, + "reserved_bytes_peak": 111904030720, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:44.461701Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105791051264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 6675008512, + "inactive_split_bytes_peak": 9434158080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 111490891776, + "reserved_bytes_peak": 111490891776, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T13:58:43.446033Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8145112064, + "inactive_split_bytes_current_delta": 1470103552, + "inactive_split_bytes_peak": 9434158080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 111904030720, + "reserved_bytes_current_delta": 413138944, + "reserved_bytes_peak": 111904030720, + "reserved_bytes_peak_delta": 413138944, + "segment_count_current": 492, + "segment_count_current_delta": 1, + "segment_count_peak": 492, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T135844462115Z.pickle", + "timestamp_utc": "2026-08-01T13:58:44.570779Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T135844462115Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T135844462115Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f4cd76967f8bf2db45c3d048adcd985641efe0aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T135844462115Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:410dd10e8e836356e0408db8bc2f3770aa32ae5dda85d6281ff4698a9a2f7d15 +size 6084874 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T135845749078Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T135845749078Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0aef7ae8034f4a424b47350aa682bbe9007b01ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T135845749078Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418379264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 7092341760, + "inactive_split_bytes_peak": 9434158080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 111904030720, + "reserved_bytes_peak": 111904030720, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:45.748680Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105791051264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 8145112064, + "inactive_split_bytes_peak": 9434158080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 111904030720, + "reserved_bytes_peak": 111904030720, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:44.586300Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7092341760, + "inactive_split_bytes_current_delta": -1052770304, + "inactive_split_bytes_peak": 9434158080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 111904030720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111904030720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 492, + "segment_count_current_delta": 0, + "segment_count_peak": 492, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T135845749078Z.pickle", + "timestamp_utc": "2026-08-01T13:58:45.857245Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T135845749078Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T135845749078Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..339b95e2e6b55b3e2a62e6a86c27cad23f0efd0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T135845749078Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:867f4ac9bf7e3a8f9998d85d8c982768a515e5619984cd95e89336d46744a3e6 +size 6618686 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T135846884725Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T135846884725Z.json new file mode 100644 index 0000000000000000000000000000000000000000..782f4f350920a4ec294c995f8739558a9c71848c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T135846884725Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106418379264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 7148964864, + "inactive_split_bytes_peak": 9434158080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 111904030720, + "reserved_bytes_peak": 111904030720, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:46.884299Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105791051264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 7092341760, + "inactive_split_bytes_peak": 9434158080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 111904030720, + "reserved_bytes_peak": 111904030720, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:45.872797Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7148964864, + "inactive_split_bytes_current_delta": 56623104, + "inactive_split_bytes_peak": 9434158080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 111904030720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111904030720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 492, + "segment_count_current_delta": 0, + "segment_count_peak": 492, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T135846884725Z.pickle", + "timestamp_utc": "2026-08-01T13:58:46.991920Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T135846884725Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T135846884725Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..959ec8f3415f2145f7b61bc22f74e620a78cb137 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T135846884725Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25876dd6cd39cd9df79cf93bd767e818cf807d89b0f9680d630ed6c3cf75afcf +size 6618765 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T135941162736Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T135941162736Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1914547a7cecefa67a69fab9cd5c1d4300e3e9be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T135941162736Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 106418379264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 6614191104, + "inactive_split_bytes_peak": 9434158080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 111904030720, + "reserved_bytes_peak": 111904030720, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:59:41.162153Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105791051264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106418379264, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105791051264, + "device": 5, + "inactive_split_bytes_current": 7148964864, + "inactive_split_bytes_peak": 9434158080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 111904030720, + "reserved_bytes_peak": 111904030720, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:47.007094Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6614191104, + "inactive_split_bytes_current_delta": -534773760, + "inactive_split_bytes_peak": 9434158080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 111904030720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111904030720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 492, + "segment_count_current_delta": 0, + "segment_count_peak": 492, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T135941162736Z.pickle", + "timestamp_utc": "2026-08-01T13:59:41.380981Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T135941162736Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T135941162736Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dc44e5370897715b28c4ae58fc3d9726700b6b26 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T135941162736Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2118944a1efe7326201b3a0dd7f7721648cc873b32a4841a8cb600d9f5a6a319 +size 10533173 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T135837143286Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T135837143286Z.json new file mode 100644 index 0000000000000000000000000000000000000000..243a0df913d0d3aa0914f2d4b7a5d2f0cb0d79b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T135837143286Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 82423007232, + "active_bytes_peak_delta": 46956670976, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82423007232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82144461824, + "device": 6, + "inactive_split_bytes_current": 6849072128, + "inactive_split_bytes_peak": 7893063680, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 100384374784, + "reserved_bytes_peak": 100384374784, + "segment_count_current": 464, + "segment_count_peak": 464, + "timestamp_utc": "2026-08-01T13:58:37.142668Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 82144461824, + "allocated_bytes_peak_delta": 46678125568, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:57:29.296883Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6849072128, + "inactive_split_bytes_current_delta": 6716253184, + "inactive_split_bytes_peak": 7893063680, + "inactive_split_bytes_peak_delta": 7732677632, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 100384374784, + "reserved_bytes_current_delta": 61140369408, + "reserved_bytes_peak": 100384374784, + "reserved_bytes_peak_delta": 61140369408, + "segment_count_current": 464, + "segment_count_current_delta": 393, + "segment_count_peak": 464, + "segment_count_peak_delta": 393, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T135837143286Z.pickle", + "timestamp_utc": "2026-08-01T13:58:37.261749Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T135837143286Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T135837143286Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eb4122756a3e2bd7819968ec4e9e45775d4d1cd4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T135837143286Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:001dda6920438792129e1d680d1ae5aa6699f9ba50162916ba73254fd04178dd +size 5942796 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T135838280929Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T135838280929Z.json new file mode 100644 index 0000000000000000000000000000000000000000..52c84cfd99c1800b6e1302269afc8ff6581ac583 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T135838280929Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106415344128, + "active_bytes_peak_delta": 23992336896, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105785132032, + "device": 6, + "inactive_split_bytes_current": 6557568000, + "inactive_split_bytes_peak": 8600455168, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 110756888576, + "reserved_bytes_peak": 110756888576, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:38.280511Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105785132032, + "allocated_bytes_peak_delta": 23640670208, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 82423007232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 82144461824, + "device": 6, + "inactive_split_bytes_current": 6849072128, + "inactive_split_bytes_peak": 7893063680, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 100384374784, + "reserved_bytes_peak": 100384374784, + "segment_count_current": 464, + "segment_count_peak": 464, + "timestamp_utc": "2026-08-01T13:58:37.280376Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6557568000, + "inactive_split_bytes_current_delta": -291504128, + "inactive_split_bytes_peak": 8600455168, + "inactive_split_bytes_peak_delta": 707391488, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 110756888576, + "reserved_bytes_current_delta": 10372513792, + "reserved_bytes_peak": 110756888576, + "reserved_bytes_peak_delta": 10372513792, + "segment_count_current": 492, + "segment_count_current_delta": 28, + "segment_count_peak": 492, + "segment_count_peak_delta": 28, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T135838280929Z.pickle", + "timestamp_utc": "2026-08-01T13:58:38.398336Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T135838280929Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T135838280929Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27ec6154e041280bd8f7269e0d8668f8156e3d93 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T135838280929Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9898d44a33435d75f72385fcb10bdf0414b16169c045c9932f0f709814a6c1fc +size 6101925 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T135839419408Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T135839419408Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aca4d024429fbbf65e9a73967f89a9ea81faf56c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T135839419408Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106415344128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105786311680, + "device": 6, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_peak": 8600455168, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 110756888576, + "reserved_bytes_peak": 110756888576, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:39.418998Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105786311680, + "allocated_bytes_peak_delta": 1179648, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105785132032, + "device": 6, + "inactive_split_bytes_current": 6557568000, + "inactive_split_bytes_peak": 8600455168, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 110756888576, + "reserved_bytes_peak": 110756888576, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:38.414451Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_current_delta": 352321536, + "inactive_split_bytes_peak": 8600455168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 110756888576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110756888576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 492, + "segment_count_current_delta": 0, + "segment_count_peak": 492, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T135839419408Z.pickle", + "timestamp_utc": "2026-08-01T13:58:39.530937Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T135839419408Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T135839419408Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f4083d98b79a30fd5b5a739a75190782470432d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T135839419408Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd8d365c29dddeb273b6bc1b5f45f6cbf364c0afffab3d3b5b06d3d546133e80 +size 6622151 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T135840796991Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T135840796991Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ebabe8aa62a91859abd5f2a4079d1571c849c1b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T135840796991Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106415344128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788802048, + "device": 6, + "inactive_split_bytes_current": 6440127488, + "inactive_split_bytes_peak": 8600455168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 110756888576, + "reserved_bytes_peak": 110756888576, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:40.796569Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788802048, + "allocated_bytes_peak_delta": 2490368, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105786311680, + "device": 6, + "inactive_split_bytes_current": 6909889536, + "inactive_split_bytes_peak": 8600455168, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 110756888576, + "reserved_bytes_peak": 110756888576, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:39.546687Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6440127488, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 8600455168, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 110756888576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 110756888576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 492, + "segment_count_current_delta": 0, + "segment_count_peak": 492, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T135840796991Z.pickle", + "timestamp_utc": "2026-08-01T13:58:40.908100Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T135840796991Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T135840796991Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c3374bb759cbc19e3f482e3823e86e1004ed1b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T135840796991Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:628ae30784dc97b7c9f236ebfb7081d0b3fbe642060a8b948938d112685475af +size 6621947 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T135842181531Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T135842181531Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4aa03afcc67b9657ced69caa035fcf021755f2ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T135842181531Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106415344128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788802048, + "device": 6, + "inactive_split_bytes_current": 6261869568, + "inactive_split_bytes_peak": 9251841024, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112696754176, + "reserved_bytes_peak": 112696754176, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:58:42.181111Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788802048, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788802048, + "device": 6, + "inactive_split_bytes_current": 6440127488, + "inactive_split_bytes_peak": 8600455168, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 110756888576, + "reserved_bytes_peak": 110756888576, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T13:58:40.923735Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6261869568, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 9251841024, + "inactive_split_bytes_peak_delta": 651385856, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112696754176, + "reserved_bytes_current_delta": 1939865600, + "reserved_bytes_peak": 112696754176, + "reserved_bytes_peak_delta": 1939865600, + "segment_count_current": 497, + "segment_count_current_delta": 5, + "segment_count_peak": 497, + "segment_count_peak_delta": 5, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T135842181531Z.pickle", + "timestamp_utc": "2026-08-01T13:58:42.291510Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T135842181531Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T135842181531Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c6157ee958adad8bfef8560039900de9c3a52a11 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T135842181531Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b81c982442e22e79497b4087220c2defff50cdb2dcb1085140a98768ce3bf6a1 +size 6099674 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T135843321313Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T135843321313Z.json new file mode 100644 index 0000000000000000000000000000000000000000..123f7e63f4d829c876b49a885cb93b9c58813eb1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T135843321313Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106415344128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788802048, + "device": 6, + "inactive_split_bytes_current": 6849072128, + "inactive_split_bytes_peak": 9251841024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112696754176, + "reserved_bytes_peak": 112696754176, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:58:43.320907Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105788802048, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788802048, + "device": 6, + "inactive_split_bytes_current": 6261869568, + "inactive_split_bytes_peak": 9251841024, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112696754176, + "reserved_bytes_peak": 112696754176, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:58:42.306859Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6849072128, + "inactive_split_bytes_current_delta": 587202560, + "inactive_split_bytes_peak": 9251841024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112696754176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112696754176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 497, + "segment_count_current_delta": 0, + "segment_count_peak": 497, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T135843321313Z.pickle", + "timestamp_utc": "2026-08-01T13:58:43.429114Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T135843321313Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T135843321313Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ad1ce3c1a4be9f4968f1a44714e5e619b5e1025 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T135843321313Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c657a4acd77cc9e67ec0c0d41e9a154ab17513cdbdcb1d17184dea023d141fe8 +size 6621653 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T135844462123Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T135844462123Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b03ac576caad32f831df798666a7366ee07c516a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T135844462123Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106415344128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105790244352, + "device": 6, + "inactive_split_bytes_current": 6379310080, + "inactive_split_bytes_peak": 9251841024, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112696754176, + "reserved_bytes_peak": 112696754176, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:58:44.461714Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105790244352, + "allocated_bytes_peak_delta": 1442304, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105788802048, + "device": 6, + "inactive_split_bytes_current": 6849072128, + "inactive_split_bytes_peak": 9251841024, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112696754176, + "reserved_bytes_peak": 112696754176, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:58:43.444562Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6379310080, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 9251841024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112696754176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112696754176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 497, + "segment_count_current_delta": 0, + "segment_count_peak": 497, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T135844462123Z.pickle", + "timestamp_utc": "2026-08-01T13:58:44.568873Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T135844462123Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T135844462123Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..68a84997d4686a2429ef8dd53a64b88989761a16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T135844462123Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0a0f6cf995cec560ddb56130b23fc93bdd8d4d50757e370e3b1c61f7a672475 +size 6621768 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T135845749023Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T135845749023Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d752721ab4ddbadf1d1a39f7fa35d191317e067 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T135845749023Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106415344128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105790244352, + "device": 6, + "inactive_split_bytes_current": 6731631616, + "inactive_split_bytes_peak": 9251841024, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112696754176, + "reserved_bytes_peak": 112696754176, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:58:45.748631Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105790244352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105790244352, + "device": 6, + "inactive_split_bytes_current": 6379310080, + "inactive_split_bytes_peak": 9251841024, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112696754176, + "reserved_bytes_peak": 112696754176, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:58:44.584280Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6731631616, + "inactive_split_bytes_current_delta": 352321536, + "inactive_split_bytes_peak": 9251841024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112696754176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112696754176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 497, + "segment_count_current_delta": 0, + "segment_count_peak": 497, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T135845749023Z.pickle", + "timestamp_utc": "2026-08-01T13:58:45.855905Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T135845749023Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T135845749023Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..61fd719ab41acab98c8172c9bea650d2f5c8c159 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T135845749023Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f602f0922539d793765b2755ae8f2a0c9ae0bc8f501a2b46a3d58a27819f28bb +size 6621013 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T135846884683Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T135846884683Z.json new file mode 100644 index 0000000000000000000000000000000000000000..60f2f245b69f6c68b5eafb78eaa0de426c31b650 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T135846884683Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106415344128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105790278144, + "device": 6, + "inactive_split_bytes_current": 6675008512, + "inactive_split_bytes_peak": 9251841024, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112696754176, + "reserved_bytes_peak": 112696754176, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:58:46.884278Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105790278144, + "allocated_bytes_peak_delta": 33792, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105790244352, + "device": 6, + "inactive_split_bytes_current": 6731631616, + "inactive_split_bytes_peak": 9251841024, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112696754176, + "reserved_bytes_peak": 112696754176, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:58:45.871410Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6675008512, + "inactive_split_bytes_current_delta": -56623104, + "inactive_split_bytes_peak": 9251841024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112696754176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112696754176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 497, + "segment_count_current_delta": 0, + "segment_count_peak": 497, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T135846884683Z.pickle", + "timestamp_utc": "2026-08-01T13:58:46.990686Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T135846884683Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T135846884683Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..49e2402ff854dbe52dca8ba6272ae334d644fbfd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T135846884683Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5391a570ed3e8eaa0aca2362d0a2d5bda93591d80edb99c30de4448dce0959a4 +size 6620342 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T135941162596Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T135941162596Z.json new file mode 100644 index 0000000000000000000000000000000000000000..50d4040f882877ba696e800e2965251a9e07cba5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T135941162596Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 106415344128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 105790278144, + "device": 6, + "inactive_split_bytes_current": 6849072128, + "inactive_split_bytes_peak": 9251841024, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112696754176, + "reserved_bytes_peak": 112696754176, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:59:41.162115Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105790278144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 106415344128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 105790278144, + "device": 6, + "inactive_split_bytes_current": 6675008512, + "inactive_split_bytes_peak": 9251841024, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112696754176, + "reserved_bytes_peak": 112696754176, + "segment_count_current": 497, + "segment_count_peak": 497, + "timestamp_utc": "2026-08-01T13:58:47.005908Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6849072128, + "inactive_split_bytes_current_delta": 174063616, + "inactive_split_bytes_peak": 9251841024, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112696754176, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112696754176, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 497, + "segment_count_current_delta": 0, + "segment_count_peak": 497, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T135941162596Z.pickle", + "timestamp_utc": "2026-08-01T13:59:41.364205Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T135941162596Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T135941162596Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ac061e8402499c2aa66a992d8158758872262b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T135941162596Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de5c24c2abd6d8e5425c423bd7360b247db4b1aa72df73f272fff5add5cb5e47 +size 10535394 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T135837143044Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T135837143044Z.json new file mode 100644 index 0000000000000000000000000000000000000000..55e2ec77c94fcd1e0ea2d28e4ef6d594b0f99d37 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T135837143044Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 37208595968, + "active_bytes_peak": 83118942208, + "active_bytes_peak_delta": 47652605952, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 83118942208, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 82144198656, + "device": 7, + "inactive_split_bytes_current": 6909889024, + "inactive_split_bytes_peak": 6909890048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 100973674496, + "reserved_bytes_peak": 100973674496, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T13:58:37.142468Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 37208595968, + "allocated_bytes_peak": 82144198656, + "allocated_bytes_peak_delta": 46677862400, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T13:57:29.297078Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6909889024, + "inactive_split_bytes_current_delta": 6777070080, + "inactive_split_bytes_peak": 6909890048, + "inactive_split_bytes_peak_delta": 6749504000, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 100973674496, + "reserved_bytes_current_delta": 61729669120, + "reserved_bytes_peak": 100973674496, + "reserved_bytes_peak_delta": 61729669120, + "segment_count_current": 467, + "segment_count_current_delta": 396, + "segment_count_peak": 467, + "segment_count_peak_delta": 396, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T135837143044Z.pickle", + "timestamp_utc": "2026-08-01T13:58:37.259167Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T135837143044Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T135837143044Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a37e6a4d63d256948d85071db5fc28abda1c735c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T135837143044Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcc238f23bf5b9c02c26318158fb2918cfe918cfe9bb6797fc8bd248b8b0fbad +size 5944994 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T135838280879Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T135838280879Z.json new file mode 100644 index 0000000000000000000000000000000000000000..557a7510eb1a23a2008ae6403e4135fd539ec858 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T135838280879Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106066231296, + "active_bytes_peak_delta": 22947289088, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106066231296, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105787948032, + "device": 7, + "inactive_split_bytes_current": 6500944384, + "inactive_split_bytes_peak": 8643227136, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 111021129728, + "reserved_bytes_peak": 111021129728, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T13:58:38.280438Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105787948032, + "allocated_bytes_peak_delta": 23643749376, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 83118942208, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 82144198656, + "device": 7, + "inactive_split_bytes_current": 6909889024, + "inactive_split_bytes_peak": 6909890048, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 100973674496, + "reserved_bytes_peak": 100973674496, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T13:58:37.276561Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6500944384, + "inactive_split_bytes_current_delta": -408944640, + "inactive_split_bytes_peak": 8643227136, + "inactive_split_bytes_peak_delta": 1733337088, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 111021129728, + "reserved_bytes_current_delta": 10047455232, + "reserved_bytes_peak": 111021129728, + "reserved_bytes_peak_delta": 10047455232, + "segment_count_current": 493, + "segment_count_current_delta": 26, + "segment_count_peak": 493, + "segment_count_peak_delta": 26, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T135838280879Z.pickle", + "timestamp_utc": "2026-08-01T13:58:38.397173Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T135838280879Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T135838280879Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e804c774800f8308ad44bd902daa96814a611b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T135838280879Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f74a5a9d3868c5f56cd2a24ad0b2e2a06f1a3a360edede9696a3f15165f44e65 +size 6626352 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T135839419392Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T135839419392Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bce30ab40204e9c8e45f07a2f01f92c25ddf6e6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T135839419392Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106067127296, + "active_bytes_peak_delta": 896000, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106067127296, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105789630464, + "device": 7, + "inactive_split_bytes_current": 6500944384, + "inactive_split_bytes_peak": 8643227136, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 111021129728, + "reserved_bytes_peak": 111021129728, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T13:58:39.418974Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105789630464, + "allocated_bytes_peak_delta": 1682432, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106066231296, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105787948032, + "device": 7, + "inactive_split_bytes_current": 6500944384, + "inactive_split_bytes_peak": 8643227136, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 111021129728, + "reserved_bytes_peak": 111021129728, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T13:58:38.411281Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6500944384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8643227136, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 111021129728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111021129728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 493, + "segment_count_current_delta": 0, + "segment_count_peak": 493, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T135839419392Z.pickle", + "timestamp_utc": "2026-08-01T13:58:39.526686Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T135839419392Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T135839419392Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a60e2edf4b94cd33d3a592e434c788f935648d20 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T135839419392Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee80c1ed23abba1c51b2303798be62ecdcba9678b4672bdc6439082842121dd2 +size 6623963 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T135840796965Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T135840796965Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7ecd9175c98e4a19c12aea4730b573bfc6cf3312 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T135840796965Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106069597184, + "active_bytes_peak_delta": 2469888, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106069597184, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 7440468480, + "inactive_split_bytes_peak": 8643227136, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 111021129728, + "reserved_bytes_peak": 111021129728, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T13:58:40.796559Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792100352, + "allocated_bytes_peak_delta": 2469888, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106067127296, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105789630464, + "device": 7, + "inactive_split_bytes_current": 6500944384, + "inactive_split_bytes_peak": 8643227136, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 111021129728, + "reserved_bytes_peak": 111021129728, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T13:58:39.540354Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7440468480, + "inactive_split_bytes_current_delta": 939524096, + "inactive_split_bytes_peak": 8643227136, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 111021129728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 111021129728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 493, + "segment_count_current_delta": 0, + "segment_count_peak": 493, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T135840796965Z.pickle", + "timestamp_utc": "2026-08-01T13:58:40.905464Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T135840796965Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T135840796965Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e417767a07e4ffec1e60874754223b944d0f6af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T135840796965Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa7d734d14b15d6d87970a981a097d39d33c6a3a71c843ae0af312a214a88f45 +size 6623594 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T135842181490Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T135842181490Z.json new file mode 100644 index 0000000000000000000000000000000000000000..57ce7ab4dd496d2462781532e4e0f923040a4a89 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T135842181490Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106069597184, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106069597184, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 6731631104, + "inactive_split_bytes_peak": 9781167616, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112256352256, + "reserved_bytes_peak": 112256352256, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:42.181071Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792100352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106069597184, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 7440468480, + "inactive_split_bytes_peak": 8643227136, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 111021129728, + "reserved_bytes_peak": 111021129728, + "segment_count_current": 493, + "segment_count_peak": 493, + "timestamp_utc": "2026-08-01T13:58:40.919375Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6731631104, + "inactive_split_bytes_current_delta": -708837376, + "inactive_split_bytes_peak": 9781167616, + "inactive_split_bytes_peak_delta": 1137940480, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112256352256, + "reserved_bytes_current_delta": 1235222528, + "reserved_bytes_peak": 112256352256, + "reserved_bytes_peak_delta": 1235222528, + "segment_count_current": 496, + "segment_count_current_delta": 3, + "segment_count_peak": 496, + "segment_count_peak_delta": 3, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T135842181490Z.pickle", + "timestamp_utc": "2026-08-01T13:58:42.289340Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T135842181490Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T135842181490Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ba6c89702fc29b371e47f1d7d694054589856937 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T135842181490Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a022cf90ee399c44e999969c7ec129888f890cbd625b51d27f9b1d2bf15d985b +size 6099158 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T135843321269Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T135843321269Z.json new file mode 100644 index 0000000000000000000000000000000000000000..320e26497e7b1f4da028d69de3d531b8c04e3d49 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T135843321269Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106069708288, + "active_bytes_peak_delta": 111104, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106069708288, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 7203490304, + "inactive_split_bytes_peak": 9781167616, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112256352256, + "reserved_bytes_peak": 112256352256, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:43.320865Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792100352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106069597184, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 6731631104, + "inactive_split_bytes_peak": 9781167616, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112256352256, + "reserved_bytes_peak": 112256352256, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:42.303235Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7203490304, + "inactive_split_bytes_current_delta": 471859200, + "inactive_split_bytes_peak": 9781167616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112256352256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112256352256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 496, + "segment_count_current_delta": 0, + "segment_count_peak": 496, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T135843321269Z.pickle", + "timestamp_utc": "2026-08-01T13:58:43.429875Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T135843321269Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T135843321269Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e306e4cc3a7c7cac5ceda634b83557202e61fc2c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T135843321269Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36051652bc4d8a906580f0f63ddc459059a3632c11b94d151f9ade99c5a43429 +size 6620374 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T135844462122Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T135844462122Z.json new file mode 100644 index 0000000000000000000000000000000000000000..38cd16ae011a701d388f0c7db18dcb7a9e616de2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T135844462122Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106069708288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106069708288, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 6440126976, + "inactive_split_bytes_peak": 9781167616, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112256352256, + "reserved_bytes_peak": 112256352256, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:44.461722Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792100352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106069708288, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 7203490304, + "inactive_split_bytes_peak": 9781167616, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112256352256, + "reserved_bytes_peak": 112256352256, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:43.445851Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6440126976, + "inactive_split_bytes_current_delta": -763363328, + "inactive_split_bytes_peak": 9781167616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112256352256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112256352256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 496, + "segment_count_current_delta": 0, + "segment_count_peak": 496, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T135844462122Z.pickle", + "timestamp_utc": "2026-08-01T13:58:44.567984Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T135844462122Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T135844462122Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..31aa8e08550481e0f65fa1f8e5a3ae54b27435f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T135844462122Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cc8f345829b78546a370851ce39ebba0eaaac26f53061c43462596a0dad83cd +size 6619815 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T135845749068Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T135845749068Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd48c528f0189ad1eda81a6977b08f3844c067e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T135845749068Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106069708288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106069708288, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 7266404864, + "inactive_split_bytes_peak": 9781167616, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112256352256, + "reserved_bytes_peak": 112256352256, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:45.748668Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792100352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106069708288, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 6440126976, + "inactive_split_bytes_peak": 9781167616, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112256352256, + "reserved_bytes_peak": 112256352256, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:44.581443Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7266404864, + "inactive_split_bytes_current_delta": 826277888, + "inactive_split_bytes_peak": 9781167616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112256352256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112256352256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 496, + "segment_count_current_delta": 0, + "segment_count_peak": 496, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T135845749068Z.pickle", + "timestamp_utc": "2026-08-01T13:58:45.854972Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T135845749068Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T135845749068Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..08a93791516c6784f5c8d8d740e5b8cd13e6ded1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T135845749068Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1f4353f171f09082eac74923024e94e8c67e1b36f9a7573b4049357685880f0 +size 6619662 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T135846884738Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T135846884738Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1cbd74391bc36dfd6c090906587755b0b51216e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T135846884738Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 106069708288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106069708288, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 6914083328, + "inactive_split_bytes_peak": 9781167616, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112256352256, + "reserved_bytes_peak": 112256352256, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:46.884325Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 105792100352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106069708288, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 7266404864, + "inactive_split_bytes_peak": 9781167616, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112256352256, + "reserved_bytes_peak": 112256352256, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:45.868693Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6914083328, + "inactive_split_bytes_current_delta": -352321536, + "inactive_split_bytes_peak": 9781167616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112256352256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112256352256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 496, + "segment_count_current_delta": 0, + "segment_count_peak": 496, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T135846884738Z.pickle", + "timestamp_utc": "2026-08-01T13:58:46.986879Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T135846884738Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T135846884738Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ec3f413309ab0aaa7db94b5d240d5d4937c54f10 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T135846884738Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e630a89f68a52c68493dd24543ff03f271f580dfeb39e24aaae2d9d5df3722ac +size 6619633 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T135941162602Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T135941162602Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aeb6f4c39de8a829018e73755f95e102b6e52adb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T135941162602Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165762048, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 106069708288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165762048, + "active_bytes_peak": 106069708288, + "allocated_bytes_current": 61165762048, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 6261869056, + "inactive_split_bytes_peak": 9781167616, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112256352256, + "reserved_bytes_peak": 112256352256, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:59:41.162117Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165762048, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 105792100352, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 106069708288, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 105792100352, + "device": 7, + "inactive_split_bytes_current": 6914083328, + "inactive_split_bytes_peak": 9781167616, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112256352256, + "reserved_bytes_peak": 112256352256, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T13:58:46.999967Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6261869056, + "inactive_split_bytes_current_delta": -652214272, + "inactive_split_bytes_peak": 9781167616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112256352256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112256352256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 496, + "segment_count_current_delta": 0, + "segment_count_peak": 496, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T135941162602Z.pickle", + "timestamp_utc": "2026-08-01T13:59:41.360790Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T135941162602Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T135941162602Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c7c0233d908218a5f21b2a7eed266ea661d5b2c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T135941162602Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fd1d3ad90a018169c922815c33b79eb1b32e149c048372ef637d2271181f75a +size 10534304 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..e068037321db74a57ed701dd268f3de82186d973 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 4096 +capacity: 1.00 +expert_dispatch_overlap: off +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6036 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 225 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T140226665228Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T140226665228Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4916d11bff2e201d42a8d2e5d1924c52b141f5e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T140226665228Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 31622347776, + "active_bytes_peak": 87091222528, + "active_bytes_peak_delta": 51624886272, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 87091222528, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 87091222528, + "device": 0, + "inactive_split_bytes_current": 7435709440, + "inactive_split_bytes_peak": 9908950016, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 94183096320, + "reserved_bytes_peak": 94183096320, + "segment_count_current": 511, + "segment_count_peak": 511, + "timestamp_utc": "2026-08-01T14:02:26.664833Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 31622347776, + "allocated_bytes_peak": 87091222528, + "allocated_bytes_peak_delta": 51624886272, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T14:01:19.781044Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7435709440, + "inactive_split_bytes_current_delta": 7300793344, + "inactive_split_bytes_peak": 9908950016, + "inactive_split_bytes_peak_delta": 9748318208, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 94183096320, + "reserved_bytes_current_delta": 54936993792, + "reserved_bytes_peak": 94183096320, + "reserved_bytes_peak_delta": 54936993792, + "segment_count_current": 511, + "segment_count_current_delta": 439, + "segment_count_peak": 511, + "segment_count_peak_delta": 439, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T140226665228Z.pickle", + "timestamp_utc": "2026-08-01T14:02:26.775215Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T140226665228Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T140226665228Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c7022fd7108dd6c88c2a648c488c09068e56cca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T140226665228Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe4e8080363284d17f1ae5e15aa63e1af6ead84042be2bb3bfe9a89686ee88d8 +size 5394606 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T140227845665Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T140227845665Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f4c7eb6526762e897f7228d1b8e0be860d408b36 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T140227845665Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110724491776, + "active_bytes_peak_delta": 23633269248, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110724491776, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110724491776, + "device": 0, + "inactive_split_bytes_current": 6506671104, + "inactive_split_bytes_peak": 9908950016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:27.845255Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110724491776, + "allocated_bytes_peak_delta": 23633269248, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 87091222528, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 87091222528, + "device": 0, + "inactive_split_bytes_current": 7435709440, + "inactive_split_bytes_peak": 9908950016, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 94183096320, + "reserved_bytes_peak": 94183096320, + "segment_count_current": 511, + "segment_count_peak": 511, + "timestamp_utc": "2026-08-01T14:02:26.792890Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6506671104, + "inactive_split_bytes_current_delta": -929038336, + "inactive_split_bytes_peak": 9908950016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 18790481920, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 18790481920, + "segment_count_current": 596, + "segment_count_current_delta": 85, + "segment_count_peak": 596, + "segment_count_peak_delta": 85, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T140227845665Z.pickle", + "timestamp_utc": "2026-08-01T14:02:27.956065Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T140227845665Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T140227845665Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b8e3ef2120ab49a2e8c861363d820a604f1e801e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T140227845665Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:236f53b3544771bf41ec67726b8cd269f197a6aa7ae3dd0e80a55a688625d194 +size 5621704 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T140228981467Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T140228981467Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96254f4696c87add6d3a16969743b0a5e3c97819 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T140228981467Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110724491776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110724491776, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110724491776, + "device": 0, + "inactive_split_bytes_current": 5921565696, + "inactive_split_bytes_peak": 9908950016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:28.981077Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110724491776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110724491776, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110724491776, + "device": 0, + "inactive_split_bytes_current": 6506671104, + "inactive_split_bytes_peak": 9908950016, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:27.969455Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5921565696, + "inactive_split_bytes_current_delta": -585105408, + "inactive_split_bytes_peak": 9908950016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T140228981467Z.pickle", + "timestamp_utc": "2026-08-01T14:02:29.082436Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T140228981467Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T140228981467Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f63a939ccad7062880999bf1985641ae6db6dc0a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T140228981467Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:128e26e8c34a90dfe7f838ba95389ab45e6a73da12d5fdeb5ef1c89a497c4a7e +size 5624768 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T140230132324Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T140230132324Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a57d787473521ac28c0e3c0230f7388fd9184e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T140230132324Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110724491776, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110724491776, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110724491776, + "device": 0, + "inactive_split_bytes_current": 7567830016, + "inactive_split_bytes_peak": 9908950016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:30.131902Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110724491776, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110724491776, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110724491776, + "device": 0, + "inactive_split_bytes_current": 5921565696, + "inactive_split_bytes_peak": 9908950016, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:29.095571Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7567830016, + "inactive_split_bytes_current_delta": 1646264320, + "inactive_split_bytes_peak": 9908950016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T140230132324Z.pickle", + "timestamp_utc": "2026-08-01T14:02:30.236524Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T140230132324Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T140230132324Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5eab78833027019226b76ce6848eacb19ba860aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T140230132324Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86b769e006e744b981e89f766f895ab3c63bec7f16b9e7cc9688e99bc0a65bfd +size 5622595 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T140231554691Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T140231554691Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d34b4b466a6e7e728c76da34e8d2519f65e4445 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T140231554691Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110730648576, + "active_bytes_peak_delta": 6156800, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110730648576, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110730648576, + "device": 0, + "inactive_split_bytes_current": 7689464832, + "inactive_split_bytes_peak": 9908950016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:31.554292Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110730648576, + "allocated_bytes_peak_delta": 6156800, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110724491776, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110724491776, + "device": 0, + "inactive_split_bytes_current": 7567830016, + "inactive_split_bytes_peak": 9908950016, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:30.249965Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7689464832, + "inactive_split_bytes_current_delta": 121634816, + "inactive_split_bytes_peak": 9908950016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T140231554691Z.pickle", + "timestamp_utc": "2026-08-01T14:02:31.659575Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T140231554691Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T140231554691Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..194de454d673b9bea7f3b0478df5828699972772 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T140231554691Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4d1d73d07756905fd7a326bb1e2c040fe19a9b757d64bb94fa4e4d954af36a6 +size 5625025 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T140232705907Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T140232705907Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ce09bfba7b0b59346e3f3f0cb19ef98138ef3403 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T140232705907Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110730648576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110730648576, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110730648576, + "device": 0, + "inactive_split_bytes_current": 7572024320, + "inactive_split_bytes_peak": 9908950016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:32.705514Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110730648576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110730648576, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110730648576, + "device": 0, + "inactive_split_bytes_current": 7689464832, + "inactive_split_bytes_peak": 9908950016, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:31.672660Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7572024320, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 9908950016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T140232705907Z.pickle", + "timestamp_utc": "2026-08-01T14:02:32.803378Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T140232705907Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T140232705907Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..96a45b28498e9a45f346abd5eb43d9a406f2629d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T140232705907Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c5e5b8d9742903f1ee9a35a5123bd14851c04892c9da6be423f262c7bbd6f07 +size 5620755 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T140233851336Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T140233851336Z.json new file mode 100644 index 0000000000000000000000000000000000000000..da75d07c6a1ff1bd1d767d2b8ecca8eaf458ca39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T140233851336Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110730648576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110730648576, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110730648576, + "device": 0, + "inactive_split_bytes_current": 6275984384, + "inactive_split_bytes_peak": 9908950016, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:33.850933Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110730648576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110730648576, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110730648576, + "device": 0, + "inactive_split_bytes_current": 7572024320, + "inactive_split_bytes_peak": 9908950016, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:32.815981Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6275984384, + "inactive_split_bytes_current_delta": -1296039936, + "inactive_split_bytes_peak": 9908950016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T140233851336Z.pickle", + "timestamp_utc": "2026-08-01T14:02:33.950757Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T140233851336Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T140233851336Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b76bd6ec303917223e0d55db2df6cb19b54f0ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T140233851336Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40498c4504b6d113ce0c2aff0843e65cf307010260030831e65aa7486b6a4330 +size 5620380 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T140235008193Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T140235008193Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f26e6755a424d6bb4cbc0a8034b1dcc6105c0ed4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T140235008193Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110730648576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110730648576, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110730648576, + "device": 0, + "inactive_split_bytes_current": 6095629312, + "inactive_split_bytes_peak": 9908950016, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:35.007785Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110730648576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110730648576, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110730648576, + "device": 0, + "inactive_split_bytes_current": 6275984384, + "inactive_split_bytes_peak": 9908950016, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:33.963475Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6095629312, + "inactive_split_bytes_current_delta": -180355072, + "inactive_split_bytes_peak": 9908950016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T140235008193Z.pickle", + "timestamp_utc": "2026-08-01T14:02:35.108186Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T140235008193Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T140235008193Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d0b459fca7b4e51319ccf19fe699efd88b494c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T140235008193Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de6bae97bc501e1d1182cdc47b3e3870125325569393ffe10afd04ea03121f4e +size 5619851 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T140236157672Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T140236157672Z.json new file mode 100644 index 0000000000000000000000000000000000000000..286e7dfdc0072ebf81d16b8840c414f85c9ccd7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T140236157672Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853803008, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110730648576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110730648576, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110730648576, + "device": 0, + "inactive_split_bytes_current": 6632500224, + "inactive_split_bytes_peak": 9908950016, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:36.157277Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853803008, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110730648576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110730648576, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110730648576, + "device": 0, + "inactive_split_bytes_current": 6095629312, + "inactive_split_bytes_peak": 9908950016, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:35.120849Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6632500224, + "inactive_split_bytes_current_delta": 536870912, + "inactive_split_bytes_peak": 9908950016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T140236157672Z.pickle", + "timestamp_utc": "2026-08-01T14:02:36.256499Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T140236157672Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T140236157672Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ecab8a0726f9150c2b40fa8447d828245f65065e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T140236157672Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e7c125513050b3d0d7adc09f42948237bc366ac2fc9e8740ed58455a52d0f7 +size 5626218 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T140334988974Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T140334988974Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef51b8496fd4a141ab4d2d0b60986c52dfb7b440 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T140334988974Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803160064, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110730648576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803160064, + "active_bytes_peak": 110730648576, + "allocated_bytes_current": 66803160064, + "allocated_bytes_peak": 110730648576, + "device": 0, + "inactive_split_bytes_current": 6194506752, + "inactive_split_bytes_peak": 9908950016, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:03:34.988509Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803160064, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110730648576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853803008, + "active_bytes_peak": 110730648576, + "allocated_bytes_current": 66853803008, + "allocated_bytes_peak": 110730648576, + "device": 0, + "inactive_split_bytes_current": 6632500224, + "inactive_split_bytes_peak": 9908950016, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:36.269083Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6194506752, + "inactive_split_bytes_current_delta": -437993472, + "inactive_split_bytes_peak": 9908950016, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T140334988974Z.pickle", + "timestamp_utc": "2026-08-01T14:03:35.183509Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T140334988974Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T140334988974Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94a0ed4c2753385cb15fde9097f3b0c8e9f0b9df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T140334988974Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ffbc79466e82d915779d0112c7208f58546a04c5dd9a3408fdc0ae94acd5125 +size 10198216 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T140226665207Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T140226665207Z.json new file mode 100644 index 0000000000000000000000000000000000000000..faeea86409bc652cacfba2b858025d7fd1881707 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T140226665207Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855515136, + "active_bytes_current_delta": 31624059904, + "active_bytes_peak": 87100909056, + "active_bytes_peak_delta": 51634572800, + "after": { + "active_bytes_current": 66855515136, + "active_bytes_peak": 87100909056, + "allocated_bytes_current": 66855515136, + "allocated_bytes_peak": 87100909056, + "device": 1, + "inactive_split_bytes_current": 6460918784, + "inactive_split_bytes_peak": 7982072832, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 93778345984, + "reserved_bytes_peak": 93778345984, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T14:02:26.664634Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855515136, + "allocated_bytes_current_delta": 31624059904, + "allocated_bytes_peak": 87100909056, + "allocated_bytes_peak_delta": 51634572800, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:01:19.780544Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6460918784, + "inactive_split_bytes_current_delta": 6328099840, + "inactive_split_bytes_peak": 7982072832, + "inactive_split_bytes_peak_delta": 7821686784, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 93778345984, + "reserved_bytes_current_delta": 54534340608, + "reserved_bytes_peak": 93778345984, + "reserved_bytes_peak_delta": 54534340608, + "segment_count_current": 515, + "segment_count_current_delta": 444, + "segment_count_peak": 515, + "segment_count_peak_delta": 444, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T140226665207Z.pickle", + "timestamp_utc": "2026-08-01T14:02:26.777017Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T140226665207Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T140226665207Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eaa831bef00e4d70d74de6acd9d42e0b3a1c6f38 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T140226665207Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e47c9ff9f79f98364ad47ca6c622960a8ddcde8c80929d4f9b9cb171bc5d6de2 +size 5390172 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T140227845676Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T140227845676Z.json new file mode 100644 index 0000000000000000000000000000000000000000..de1447e8cc871f93847b01d97f435e22bc70fe3a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T140227845676Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 110730922496, + "active_bytes_peak_delta": 23630013440, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110730922496, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110730922496, + "device": 1, + "inactive_split_bytes_current": 7116278784, + "inactive_split_bytes_peak": 7982072832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:27.845269Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 110730922496, + "allocated_bytes_peak_delta": 23630013440, + "before": { + "active_bytes_current": 66855515136, + "active_bytes_peak": 87100909056, + "allocated_bytes_current": 66855515136, + "allocated_bytes_peak": 87100909056, + "device": 1, + "inactive_split_bytes_current": 6460918784, + "inactive_split_bytes_peak": 7982072832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 93778345984, + "reserved_bytes_peak": 93778345984, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T14:02:26.796541Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7116278784, + "inactive_split_bytes_current_delta": 655360000, + "inactive_split_bytes_peak": 7982072832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 19079888896, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 19079888896, + "segment_count_current": 597, + "segment_count_current_delta": 82, + "segment_count_peak": 597, + "segment_count_peak_delta": 82, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T140227845676Z.pickle", + "timestamp_utc": "2026-08-01T14:02:27.960441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T140227845676Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T140227845676Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c35ecbab5611ca0d46e260d25b4ef131220b33f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T140227845676Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df9b941b75fe8676acd4f5810870f2ceb990e35bd75402c069425996938cfd27 +size 5619495 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T140228981577Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T140228981577Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b0357ca33642360e30bf55aa4fe9c09c2f7b73d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T140228981577Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734086144, + "active_bytes_peak_delta": 3163648, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 1, + "inactive_split_bytes_current": 6887689216, + "inactive_split_bytes_peak": 8151070208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:28.981185Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734086144, + "allocated_bytes_peak_delta": 3163648, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110730922496, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110730922496, + "device": 1, + "inactive_split_bytes_current": 7116278784, + "inactive_split_bytes_peak": 7982072832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:27.975589Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6887689216, + "inactive_split_bytes_current_delta": -228589568, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 168997376, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T140228981577Z.pickle", + "timestamp_utc": "2026-08-01T14:02:29.091330Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T140228981577Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T140228981577Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d215ca7868e429c501b0b549880fb9c1b01111e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T140228981577Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acfd372737cad3a91bb412f07a2880e0fe9e47e4281f6d0e445c4f091efb549a +size 5621061 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T140230132326Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T140230132326Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6740a11fa664e21f86e5a60f3e6ec8a28cb3d711 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T140230132326Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734086144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 1, + "inactive_split_bytes_current": 7116278784, + "inactive_split_bytes_peak": 8151070208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:30.131927Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734086144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 1, + "inactive_split_bytes_current": 6887689216, + "inactive_split_bytes_peak": 8151070208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:29.108644Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7116278784, + "inactive_split_bytes_current_delta": 228589568, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T140230132326Z.pickle", + "timestamp_utc": "2026-08-01T14:02:30.239270Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T140230132326Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T140230132326Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..391b0d92241760cc1cb1575084e738d541c66e5c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T140230132326Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4e42f7e57374fd0c8ed5bbbfd9a03a35a2ee383baae44b4bce90497aac394e0 +size 5620555 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T140231554663Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T140231554663Z.json new file mode 100644 index 0000000000000000000000000000000000000000..28272d20839ba3c84e69683077d5a8521af29acd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T140231554663Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734086144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 1, + "inactive_split_bytes_current": 6466161664, + "inactive_split_bytes_peak": 8151070208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:31.554255Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734086144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 1, + "inactive_split_bytes_current": 7116278784, + "inactive_split_bytes_peak": 8151070208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:30.254274Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6466161664, + "inactive_split_bytes_current_delta": -650117120, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T140231554663Z.pickle", + "timestamp_utc": "2026-08-01T14:02:31.659857Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T140231554663Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T140231554663Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b20e6959b5358a764ee123fe198a41929a873554 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T140231554663Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1076b06ce5571b8beb3017b5430ce19a36bd2bb8bf0bba0bcbdc3d419dfc5b94 +size 5618322 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T140232705927Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T140232705927Z.json new file mode 100644 index 0000000000000000000000000000000000000000..574086018bf72afc3f97751c49addf81cae07b73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T140232705927Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 4763136, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 1, + "inactive_split_bytes_current": 7296633856, + "inactive_split_bytes_peak": 8151070208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:32.705535Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 4763136, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 1, + "inactive_split_bytes_current": 6466161664, + "inactive_split_bytes_peak": 8151070208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:31.674303Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7296633856, + "inactive_split_bytes_current_delta": 830472192, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T140232705927Z.pickle", + "timestamp_utc": "2026-08-01T14:02:32.808116Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T140232705927Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T140232705927Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c83685ff8b48300ac24a7b74b3ab1cb807e0a344 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T140232705927Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc388b511fa3f8737dd6c82d4f98f083b3506dfbdf90b56c458dd0f0b3ebe4b2 +size 5619914 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T140233851356Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T140233851356Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f27f0fba20c303d5de6018754e06cb800072e44 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T140233851356Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 1, + "inactive_split_bytes_current": 6585699328, + "inactive_split_bytes_peak": 8151070208, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:33.850953Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 1, + "inactive_split_bytes_current": 7296633856, + "inactive_split_bytes_peak": 8151070208, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:32.822601Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6585699328, + "inactive_split_bytes_current_delta": -710934528, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T140233851356Z.pickle", + "timestamp_utc": "2026-08-01T14:02:33.952218Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T140233851356Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T140233851356Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76661304d927528dffd3ba77740f4529f636f6ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T140233851356Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c07606bb8c6ae42a35a68b743c2434d0f715a35d0cbf4c1a07db9a7dc314b5b6 +size 5618167 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T140235008240Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T140235008240Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9506777ee7fa54b21b3eb36b671798aff7f82a4b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T140235008240Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 1, + "inactive_split_bytes_current": 6719917056, + "inactive_split_bytes_peak": 8151070208, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:35.007822Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 1, + "inactive_split_bytes_current": 6585699328, + "inactive_split_bytes_peak": 8151070208, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:33.966670Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6719917056, + "inactive_split_bytes_current_delta": 134217728, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T140235008240Z.pickle", + "timestamp_utc": "2026-08-01T14:02:35.109755Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T140235008240Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T140235008240Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..70b5439860cec765eab7fa00d2bd53ea8bf696ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T140235008240Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea69ed31c0a26841f468adb643efd87987ae65f21c43ce01f66440d11e1dede4 +size 5617888 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T140236157791Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T140236157791Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7fb26dc104935b5697f64d42104c0cb9d529abff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T140236157791Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 1, + "inactive_split_bytes_current": 7533612032, + "inactive_split_bytes_peak": 8151070208, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:36.157314Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 1, + "inactive_split_bytes_current": 6719917056, + "inactive_split_bytes_peak": 8151070208, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:35.124579Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7533612032, + "inactive_split_bytes_current_delta": 813694976, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T140236157791Z.pickle", + "timestamp_utc": "2026-08-01T14:02:36.259582Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T140236157791Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T140236157791Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e8f78e567f9499a32b4c89e745a27a0314474ec7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T140236157791Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4235f896266c7597532b306156e31b13aa361c255517624f7308a1d519a8ca72 +size 5619012 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T140334988997Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T140334988997Z.json new file mode 100644 index 0000000000000000000000000000000000000000..faddae30947933b56ec1fc44c3564212df901cd9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T140334988997Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803823616, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803823616, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66803823616, + "allocated_bytes_peak": 110738849280, + "device": 1, + "inactive_split_bytes_current": 6466472960, + "inactive_split_bytes_peak": 8151070208, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:03:34.988519Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803823616, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 1, + "inactive_split_bytes_current": 7533612032, + "inactive_split_bytes_peak": 8151070208, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:36.274067Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6466472960, + "inactive_split_bytes_current_delta": -1067139072, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T140334988997Z.pickle", + "timestamp_utc": "2026-08-01T14:03:35.197718Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T140334988997Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T140334988997Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c2686c0cbf87393b3b1b36efcfea2a16b6a7eae8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T140334988997Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16eba8cc295c3fe2f1fd649a19f416e26bba0a931579ed9a35cd90dc3e142baf +size 10178405 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T140226665339Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T140226665339Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c108719d07d545dcc91b045f3c399f99d18d5731 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T140226665339Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855515136, + "active_bytes_current_delta": 31624059904, + "active_bytes_peak": 87100909056, + "active_bytes_peak_delta": 51634572800, + "after": { + "active_bytes_current": 66855515136, + "active_bytes_peak": 87100909056, + "allocated_bytes_current": 66855515136, + "allocated_bytes_peak": 87100909056, + "device": 2, + "inactive_split_bytes_current": 6463015936, + "inactive_split_bytes_peak": 7984251904, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 93778345984, + "reserved_bytes_peak": 93778345984, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T14:02:26.664776Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855515136, + "allocated_bytes_current_delta": 31624059904, + "allocated_bytes_peak": 87100909056, + "allocated_bytes_peak_delta": 51634572800, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:01:19.781376Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6463015936, + "inactive_split_bytes_current_delta": 6330196992, + "inactive_split_bytes_peak": 7984251904, + "inactive_split_bytes_peak_delta": 7823865856, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 93778345984, + "reserved_bytes_current_delta": 54534340608, + "reserved_bytes_peak": 93778345984, + "reserved_bytes_peak_delta": 54534340608, + "segment_count_current": 515, + "segment_count_current_delta": 444, + "segment_count_peak": 515, + "segment_count_peak_delta": 444, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T140226665339Z.pickle", + "timestamp_utc": "2026-08-01T14:02:26.777360Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T140226665339Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T140226665339Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..040ce96b0c1893fe156300c5a98fcf53f647a48f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T140226665339Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92fdd1435d63454ab7e86ee0f170599fe3587b1613d07f31054f7d8c6d543ca0 +size 5390219 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T140227845676Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T140227845676Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b56de9bedc80ef671eba8f6d12999a3838099177 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T140227845676Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 110730922496, + "active_bytes_peak_delta": 23630013440, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110730922496, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110730922496, + "device": 2, + "inactive_split_bytes_current": 7120473088, + "inactive_split_bytes_peak": 7984251904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112856137728, + "reserved_bytes_peak": 112856137728, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:27.845281Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 110730922496, + "allocated_bytes_peak_delta": 23630013440, + "before": { + "active_bytes_current": 66855515136, + "active_bytes_peak": 87100909056, + "allocated_bytes_current": 66855515136, + "allocated_bytes_peak": 87100909056, + "device": 2, + "inactive_split_bytes_current": 6463015936, + "inactive_split_bytes_peak": 7984251904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 93778345984, + "reserved_bytes_peak": 93778345984, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T14:02:26.794768Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7120473088, + "inactive_split_bytes_current_delta": 657457152, + "inactive_split_bytes_peak": 7984251904, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112856137728, + "reserved_bytes_current_delta": 19077791744, + "reserved_bytes_peak": 112856137728, + "reserved_bytes_peak_delta": 19077791744, + "segment_count_current": 596, + "segment_count_current_delta": 81, + "segment_count_peak": 596, + "segment_count_peak_delta": 81, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T140227845676Z.pickle", + "timestamp_utc": "2026-08-01T14:02:27.954586Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T140227845676Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T140227845676Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b76d75221b75c027a582adfbbac5e74ab1776b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T140227845676Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c12df0584b06c5fff80d5fb295e0cef8357a00cddb75c871e1e3d54155aa765 +size 5619578 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T140228981473Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T140228981473Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b78dcddf1993fd2accfab7c5d411356eccfbef60 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T140228981473Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734086144, + "active_bytes_peak_delta": 3163648, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 2, + "inactive_split_bytes_current": 6885592064, + "inactive_split_bytes_peak": 8151070208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:28.981089Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734086144, + "allocated_bytes_peak_delta": 3163648, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110730922496, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110730922496, + "device": 2, + "inactive_split_bytes_current": 7120473088, + "inactive_split_bytes_peak": 7984251904, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112856137728, + "reserved_bytes_peak": 112856137728, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:27.967882Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6885592064, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 166818304, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 597, + "segment_count_current_delta": 1, + "segment_count_peak": 597, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T140228981473Z.pickle", + "timestamp_utc": "2026-08-01T14:02:29.079774Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T140228981473Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T140228981473Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6a9ad9cce7586ef86b3798e72a0d26a3640d9c1d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T140228981473Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e26bdf0fb161d9d6e05a97496be809866bfe533a2cca925247268c6be4704cda +size 5619746 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T140230132336Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T140230132336Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f188268f7d0e4e3e3d6a02b6b563c6f777f250f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T140230132336Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734086144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 2, + "inactive_split_bytes_current": 7126764544, + "inactive_split_bytes_peak": 8151070208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:30.131930Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734086144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 2, + "inactive_split_bytes_current": 6885592064, + "inactive_split_bytes_peak": 8151070208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:29.092583Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7126764544, + "inactive_split_bytes_current_delta": 241172480, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T140230132336Z.pickle", + "timestamp_utc": "2026-08-01T14:02:30.233876Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T140230132336Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T140230132336Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d826246d274d694c43de96974b185dc9b300954c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T140230132336Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af08204cedc96903b36add5ca8c206e75263ac44ee2d9be69bd0ec29903e3a6c +size 5619319 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T140231554678Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T140231554678Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2736a86624b7cbfd67ec88ed53e0a2b284795538 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T140231554678Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734086144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 2, + "inactive_split_bytes_current": 6472453120, + "inactive_split_bytes_peak": 8151070208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:31.554267Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734086144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 2, + "inactive_split_bytes_current": 7126764544, + "inactive_split_bytes_peak": 8151070208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:30.247120Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6472453120, + "inactive_split_bytes_current_delta": -654311424, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T140231554678Z.pickle", + "timestamp_utc": "2026-08-01T14:02:31.658791Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T140231554678Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T140231554678Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0479873afc7e9333cc516ec8229ad07a9e9d7f75 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T140231554678Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed7fdec3ca6f93a6281562539f4799366077362ac5b842fe57f540326cd88c06 +size 5618461 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T140232705918Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T140232705918Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a09d2c90c70ad8c4335f863fea7e1f471813a1f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T140232705918Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 4763136, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 2, + "inactive_split_bytes_current": 7298731008, + "inactive_split_bytes_peak": 8151070208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:32.705518Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 4763136, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 2, + "inactive_split_bytes_current": 6472453120, + "inactive_split_bytes_peak": 8151070208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:31.672072Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7298731008, + "inactive_split_bytes_current_delta": 826277888, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T140232705918Z.pickle", + "timestamp_utc": "2026-08-01T14:02:32.802742Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T140232705918Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T140232705918Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f854a279cd9da69f7dd05a1762864f825f4565b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T140232705918Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76b381928be113fc59c640e547cfe2dadc2f11784d6c6ffa4319dfd70c8aa1ad +size 5619910 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T140233851351Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T140233851351Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6cdab362ef9442fffe73c163298cf8299f4f572e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T140233851351Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 2, + "inactive_split_bytes_current": 6594087936, + "inactive_split_bytes_peak": 8151070208, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:33.850947Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 2, + "inactive_split_bytes_current": 7298731008, + "inactive_split_bytes_peak": 8151070208, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:32.815218Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6594087936, + "inactive_split_bytes_current_delta": -704643072, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T140233851351Z.pickle", + "timestamp_utc": "2026-08-01T14:02:33.947179Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T140233851351Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T140233851351Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..84c5d887caebaaeb5c2cbc91935e55a7b1da983b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T140233851351Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db33b2e33ce0c9812e00f6a2402153a3817ed4b8f76520ba48eb71fd2f844e2d +size 5618177 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T140235008221Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T140235008221Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b36a98ccc2498b24c5bc0e85c13c692bfc774e22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T140235008221Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 2, + "inactive_split_bytes_current": 7061752832, + "inactive_split_bytes_peak": 8151070208, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:35.007808Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 2, + "inactive_split_bytes_current": 6594087936, + "inactive_split_bytes_peak": 8151070208, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:33.959807Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7061752832, + "inactive_split_bytes_current_delta": 467664896, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T140235008221Z.pickle", + "timestamp_utc": "2026-08-01T14:02:35.109918Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T140235008221Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T140235008221Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..339c1f1c459e3dc6c2cd8d1319dbc8906f28656c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T140235008221Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5742b732a0c614abec91d5e5e05a2816fcd347666d09c6ddf04e2aeadf1768a2 +size 5619397 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T140236157706Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T140236157706Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e6da7416a464614a5ac592f320fd13f7eca523a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T140236157706Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 2, + "inactive_split_bytes_current": 7523126272, + "inactive_split_bytes_peak": 8151070208, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:36.157302Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 2, + "inactive_split_bytes_current": 7061752832, + "inactive_split_bytes_peak": 8151070208, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:35.124604Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7523126272, + "inactive_split_bytes_current_delta": 461373440, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T140236157706Z.pickle", + "timestamp_utc": "2026-08-01T14:02:36.251812Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T140236157706Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T140236157706Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..046906ade2afccf323a6911825ae01d2cc7ec929 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T140236157706Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13b532d5da9d75423ee921c844e59eeeb234e5f68304742dcf9a0703deeed8d0 +size 5618914 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T140334989190Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T140334989190Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b8a2e1ee7708217a61f8891e3a7310e3ecdbf265 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T140334989190Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803823616, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803823616, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66803823616, + "allocated_bytes_peak": 110738849280, + "device": 2, + "inactive_split_bytes_current": 6466472960, + "inactive_split_bytes_peak": 8151070208, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:03:34.988661Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803823616, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 2, + "inactive_split_bytes_current": 7523126272, + "inactive_split_bytes_peak": 8151070208, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:36.263989Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6466472960, + "inactive_split_bytes_current_delta": -1056653312, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T140334989190Z.pickle", + "timestamp_utc": "2026-08-01T14:03:35.181800Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T140334989190Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T140334989190Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..015394b8a73c718ff27a3f93e7354ad148b3f3d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T140334989190Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fadbb580f487fb64e2058a3f313e2053148ac73ff604227eb393224ee804599 +size 9363035 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T140226665232Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T140226665232Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f8b25eca6f5fe41d20239e164579d2b2c4c964f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T140226665232Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855515136, + "active_bytes_current_delta": 31624059904, + "active_bytes_peak": 87100909056, + "active_bytes_peak_delta": 51634572800, + "after": { + "active_bytes_current": 66855515136, + "active_bytes_peak": 87100909056, + "allocated_bytes_current": 66855515136, + "allocated_bytes_peak": 87100909056, + "device": 3, + "inactive_split_bytes_current": 6463015936, + "inactive_split_bytes_peak": 7984169984, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 93778345984, + "reserved_bytes_peak": 93778345984, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T14:02:26.664641Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855515136, + "allocated_bytes_current_delta": 31624059904, + "allocated_bytes_peak": 87100909056, + "allocated_bytes_peak_delta": 51634572800, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:01:19.781243Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6463015936, + "inactive_split_bytes_current_delta": 6330196992, + "inactive_split_bytes_peak": 7984169984, + "inactive_split_bytes_peak_delta": 7823783936, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 93778345984, + "reserved_bytes_current_delta": 54534340608, + "reserved_bytes_peak": 93778345984, + "reserved_bytes_peak_delta": 54534340608, + "segment_count_current": 515, + "segment_count_current_delta": 444, + "segment_count_peak": 515, + "segment_count_peak_delta": 444, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T140226665232Z.pickle", + "timestamp_utc": "2026-08-01T14:02:26.772545Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T140226665232Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T140226665232Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..940e505d5c8dca442db56ab58a83923251db2e96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T140226665232Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3469ab66b02c5d24b08a2a3c63c1d94a1c8f4e252417f1bfd66c5fe1768ffbce +size 5390145 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T140227845664Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T140227845664Z.json new file mode 100644 index 0000000000000000000000000000000000000000..688e4c759173b32ee83f652d4318cf1d9fb4debe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T140227845664Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 110728398848, + "active_bytes_peak_delta": 23627489792, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110728398848, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110728398848, + "device": 3, + "inactive_split_bytes_current": 6822677504, + "inactive_split_bytes_peak": 8262656000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:27.845265Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 110728398848, + "allocated_bytes_peak_delta": 23627489792, + "before": { + "active_bytes_current": 66855515136, + "active_bytes_peak": 87100909056, + "allocated_bytes_current": 66855515136, + "allocated_bytes_peak": 87100909056, + "device": 3, + "inactive_split_bytes_current": 6463015936, + "inactive_split_bytes_peak": 7984169984, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 93778345984, + "reserved_bytes_peak": 93778345984, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T14:02:26.789693Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6822677504, + "inactive_split_bytes_current_delta": 359661568, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 278486016, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 18962448384, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 18962448384, + "segment_count_current": 596, + "segment_count_current_delta": 81, + "segment_count_peak": 596, + "segment_count_peak_delta": 81, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T140227845664Z.pickle", + "timestamp_utc": "2026-08-01T14:02:27.955171Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T140227845664Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T140227845664Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..960ab65f2ace90c0173b22bc5c74eb9c11e2d31a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T140227845664Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e22283b7d8b79b8e97c9b731c87c57d449eb758114b984dec0e3fe5fb21fd95 +size 5618207 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T140228981453Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T140228981453Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0b61882a8e267a2d0a5d3155cc0bd812e45a61ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T140228981453Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110732865536, + "active_bytes_peak_delta": 4466688, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 3, + "inactive_split_bytes_current": 7403588608, + "inactive_split_bytes_peak": 8262656000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:28.981058Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110732865536, + "allocated_bytes_peak_delta": 4466688, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110728398848, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110728398848, + "device": 3, + "inactive_split_bytes_current": 6822677504, + "inactive_split_bytes_peak": 8262656000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:27.968492Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7403588608, + "inactive_split_bytes_current_delta": 580911104, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T140228981453Z.pickle", + "timestamp_utc": "2026-08-01T14:02:29.082298Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T140228981453Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T140228981453Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11d8b94dd625d61dc7b371a0750ce0f2e7c4a2d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T140228981453Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baffb9a017cd6fda09051d7e497552a9791bfdda970d8615812a5806bab69f89 +size 5620836 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T140230132298Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T140230132298Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8fd297a2de4114ddafe924265f9541ca0610b091 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T140230132298Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110732865536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 3, + "inactive_split_bytes_current": 7581846528, + "inactive_split_bytes_peak": 8262656000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:30.131897Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110732865536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 3, + "inactive_split_bytes_current": 7403588608, + "inactive_split_bytes_peak": 8262656000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:29.095345Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7581846528, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T140230132298Z.pickle", + "timestamp_utc": "2026-08-01T14:02:30.236697Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T140230132298Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T140230132298Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf7ce42251c01bc74e49d848f52d2f80d5a8d69c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T140230132298Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73a044482f3d0396f587f0523e171eb434b50d5dde192e774d51920891504e59 +size 5620415 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T140231554672Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T140231554672Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc939224a66be2c10a38afe126913ab22b966fda --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T140231554672Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110732865536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 3, + "inactive_split_bytes_current": 7118375936, + "inactive_split_bytes_peak": 8262656000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:31.554254Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110732865536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 3, + "inactive_split_bytes_current": 7581846528, + "inactive_split_bytes_peak": 8262656000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:30.249891Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7118375936, + "inactive_split_bytes_current_delta": -463470592, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T140231554672Z.pickle", + "timestamp_utc": "2026-08-01T14:02:31.658373Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T140231554672Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T140231554672Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..775c381e9fc8f978ddeabfbd011e55fe476ad135 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T140231554672Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee2d459eded4ddca3318a0a35b3c4b705d5d6c6d56da27e55d415292fed56dbf +size 5620114 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T140232705971Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T140232705971Z.json new file mode 100644 index 0000000000000000000000000000000000000000..141ac47079df1daf8692154a0bcfbc94f0fde1bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T140232705971Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110732865536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 3, + "inactive_split_bytes_current": 7648955392, + "inactive_split_bytes_peak": 8262656000, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:32.705581Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110732865536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 3, + "inactive_split_bytes_current": 7118375936, + "inactive_split_bytes_peak": 8262656000, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:31.671304Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7648955392, + "inactive_split_bytes_current_delta": 530579456, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T140232705971Z.pickle", + "timestamp_utc": "2026-08-01T14:02:32.803316Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T140232705971Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T140232705971Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3eee38e1c9efe03427bc95fc2990858ba72b1aac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T140232705971Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb7c7c6803d8e2befd82866fc3fceef8c437f46f386bc5c4fd94aad06f3a0abf +size 5619655 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T140233851333Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T140233851333Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2c63e5e638eff5b9b3b89f49493fa30e620a1c77 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T140233851333Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110732865536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 3, + "inactive_split_bytes_current": 7646858240, + "inactive_split_bytes_peak": 8262656000, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:33.850929Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110732865536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 3, + "inactive_split_bytes_current": 7648955392, + "inactive_split_bytes_peak": 8262656000, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:32.815760Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7646858240, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T140233851333Z.pickle", + "timestamp_utc": "2026-08-01T14:02:33.951111Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T140233851333Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T140233851333Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a65c73527456cb51cf2c17f25e7f18cb237844cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T140233851333Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8f773651336d0c2d178b93c662749fac48a51e89db1b74e294007b062a0717c +size 5619785 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T140235008266Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T140235008266Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6498973acf6bd535af96c8e15ce25c8a3e3dc851 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T140235008266Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110737715712, + "active_bytes_peak_delta": 4850176, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110737715712, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110737715712, + "device": 3, + "inactive_split_bytes_current": 7594429440, + "inactive_split_bytes_peak": 8262656000, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:35.007793Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110737715712, + "allocated_bytes_peak_delta": 4850176, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 3, + "inactive_split_bytes_current": 7646858240, + "inactive_split_bytes_peak": 8262656000, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:33.963630Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7594429440, + "inactive_split_bytes_current_delta": -52428800, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T140235008266Z.pickle", + "timestamp_utc": "2026-08-01T14:02:35.108208Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T140235008266Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T140235008266Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c9b4b5f564c322f703bb0e587c2a3c3828a68094 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T140235008266Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73156d84724b522cec9a624b22d585d585d7407e9ae325132afb68bdbdde7d6c +size 5616864 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T140236157700Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T140236157700Z.json new file mode 100644 index 0000000000000000000000000000000000000000..81abee9231aa4af3c89f759271d9afdf5f1cf4f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T140236157700Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110737715712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110737715712, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110737715712, + "device": 3, + "inactive_split_bytes_current": 7187581952, + "inactive_split_bytes_peak": 8262656000, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:36.157289Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110737715712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110737715712, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110737715712, + "device": 3, + "inactive_split_bytes_current": 7594429440, + "inactive_split_bytes_peak": 8262656000, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:35.120663Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7187581952, + "inactive_split_bytes_current_delta": -406847488, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T140236157700Z.pickle", + "timestamp_utc": "2026-08-01T14:02:36.254819Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T140236157700Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T140236157700Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..449ed29fe2527e38bf917443d9170795847fe6f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T140236157700Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0c852569311ada808fde03dd4276fe0c5e7fa238aa7dde1991ca0315b1d15da +size 5619157 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T140334988974Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T140334988974Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b20a97735cf4c15adfe6c77abc97e2462b3a51be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T140334988974Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803823616, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110737715712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803823616, + "active_bytes_peak": 110737715712, + "allocated_bytes_current": 66803823616, + "allocated_bytes_peak": 110737715712, + "device": 3, + "inactive_split_bytes_current": 6466472960, + "inactive_split_bytes_peak": 8497203200, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:03:34.988515Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803823616, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110737715712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110737715712, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110737715712, + "device": 3, + "inactive_split_bytes_current": 7187581952, + "inactive_split_bytes_peak": 8262656000, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:36.267301Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6466472960, + "inactive_split_bytes_current_delta": -721108992, + "inactive_split_bytes_peak": 8497203200, + "inactive_split_bytes_peak_delta": 234547200, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T140334988974Z.pickle", + "timestamp_utc": "2026-08-01T14:03:35.184606Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T140334988974Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T140334988974Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e878e0600de8dc0206a401d1dad5bbbc6f0bf9f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T140334988974Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca66f5dd0e10790e8e57bcf994500411b59fc7fc34cb1b347ed6161fc6ae4454 +size 9526503 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T140226665198Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T140226665198Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34d692efc2625c0151c1eedfafc2e1f51f5d3323 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T140226665198Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855682048, + "active_bytes_current_delta": 31623784448, + "active_bytes_peak": 87097778176, + "active_bytes_peak_delta": 51630999552, + "after": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 87097778176, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 87097778176, + "device": 4, + "inactive_split_bytes_current": 6469140480, + "inactive_split_bytes_peak": 8454961152, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 94189387776, + "reserved_bytes_peak": 94189387776, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T14:02:26.664787Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855682048, + "allocated_bytes_current_delta": 31623784448, + "allocated_bytes_peak": 87097778176, + "allocated_bytes_peak_delta": 51630999552, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:01:19.780916Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6469140480, + "inactive_split_bytes_current_delta": 6336763904, + "inactive_split_bytes_peak": 8454961152, + "inactive_split_bytes_peak_delta": 8301551616, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 94189387776, + "reserved_bytes_current_delta": 54945382400, + "reserved_bytes_peak": 94189387776, + "reserved_bytes_peak_delta": 54945382400, + "segment_count_current": 516, + "segment_count_current_delta": 445, + "segment_count_peak": 516, + "segment_count_peak_delta": 445, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T140226665198Z.pickle", + "timestamp_utc": "2026-08-01T14:02:26.778390Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T140226665198Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T140226665198Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4702556db3cf5afe280fa3bbf9a4df5c5f5e7563 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T140226665198Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a82ba1e700c5f8e30d834b55dd3d411f7c59112ce30f94fa16c491ca36625dca +size 5395419 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T140227845668Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T140227845668Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0a582719e227374aba14ad57372c971911635729 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T140227845668Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855682048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110733537280, + "active_bytes_peak_delta": 23635759104, + "after": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110733537280, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110733537280, + "device": 4, + "inactive_split_bytes_current": 6639009792, + "inactive_split_bytes_peak": 8454961152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:27.845259Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855682048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110733537280, + "allocated_bytes_peak_delta": 23635759104, + "before": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 87097778176, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 87097778176, + "device": 4, + "inactive_split_bytes_current": 6469140480, + "inactive_split_bytes_peak": 8454961152, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 94189387776, + "reserved_bytes_peak": 94189387776, + "segment_count_current": 516, + "segment_count_peak": 516, + "timestamp_utc": "2026-08-01T14:02:26.797740Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6639009792, + "inactive_split_bytes_current_delta": 169869312, + "inactive_split_bytes_peak": 8454961152, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112621256704, + "reserved_bytes_current_delta": 18431868928, + "reserved_bytes_peak": 112621256704, + "reserved_bytes_peak_delta": 18431868928, + "segment_count_current": 597, + "segment_count_current_delta": 81, + "segment_count_peak": 597, + "segment_count_peak_delta": 81, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T140227845668Z.pickle", + "timestamp_utc": "2026-08-01T14:02:27.958915Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T140227845668Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T140227845668Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2948c8a989d3d8f07d7b4f17364291824c7d2324 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T140227845668Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50cfc70188f4e75c6ff6ffadd83ed8d8130aa3c51404fe06be61cbb70a1e5325 +size 5624258 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T140228981472Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T140228981472Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7d4db9e205b5a2921722ffb58ae91e115cdae375 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T140228981472Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855682048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110736945152, + "active_bytes_peak_delta": 3407872, + "after": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 7461093376, + "inactive_split_bytes_peak": 8590935040, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:28.981069Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855682048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110736945152, + "allocated_bytes_peak_delta": 3407872, + "before": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110733537280, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110733537280, + "device": 4, + "inactive_split_bytes_current": 6639009792, + "inactive_split_bytes_peak": 8454961152, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:27.976407Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7461093376, + "inactive_split_bytes_current_delta": 822083584, + "inactive_split_bytes_peak": 8590935040, + "inactive_split_bytes_peak_delta": 135973888, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112621256704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112621256704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T140228981472Z.pickle", + "timestamp_utc": "2026-08-01T14:02:29.087511Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T140228981472Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T140228981472Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..37b9b6fcea991f13f393aeba581ea526c9f38dc9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T140228981472Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:960e1aa35ce508ab7ed0276d496700fb789ea1762e5bcec9312d9675d2f0e9b5 +size 5622781 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T140230132316Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T140230132316Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a99ef671b84d3c40a358d95a30e2365f02b61e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T140230132316Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855682048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110739042304, + "active_bytes_peak_delta": 2097152, + "after": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110739042304, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110739042304, + "device": 4, + "inactive_split_bytes_current": 6880182272, + "inactive_split_bytes_peak": 8590935040, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:30.131896Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855682048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110739042304, + "allocated_bytes_peak_delta": 2097152, + "before": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 7461093376, + "inactive_split_bytes_peak": 8590935040, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:29.102223Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6880182272, + "inactive_split_bytes_current_delta": -580911104, + "inactive_split_bytes_peak": 8590935040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112621256704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112621256704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T140230132316Z.pickle", + "timestamp_utc": "2026-08-01T14:02:30.238760Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T140230132316Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T140230132316Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ecd3200cfbf402830ab59d2c7064eb5524620b9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T140230132316Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f3368cb2f8f0b31c74025174eb93df950d78f47c44f04dddd082487996d1c66 +size 5622208 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T140231554690Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T140231554690Z.json new file mode 100644 index 0000000000000000000000000000000000000000..388a628ce3bffbd3fbffa9e36c9092671bdef681 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T140231554690Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855682048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110741139456, + "active_bytes_peak_delta": 2097152, + "after": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110741139456, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110741139456, + "device": 4, + "inactive_split_bytes_current": 6532055040, + "inactive_split_bytes_peak": 8590935040, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:31.554279Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855682048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110741139456, + "allocated_bytes_peak_delta": 2097152, + "before": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110739042304, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110739042304, + "device": 4, + "inactive_split_bytes_current": 6880182272, + "inactive_split_bytes_peak": 8590935040, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:30.253724Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6532055040, + "inactive_split_bytes_current_delta": -348127232, + "inactive_split_bytes_peak": 8590935040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112621256704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112621256704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T140231554690Z.pickle", + "timestamp_utc": "2026-08-01T14:02:31.661787Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T140231554690Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T140231554690Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dd362504fba6a22e3c8cac241b9e085fbebb7cbe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T140231554690Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25241315c0537efa6b5f5026c1b4afd7af9fe0f41a8fb3804f32537337799018 +size 5621608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T140232705899Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T140232705899Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76fe6c57648b71d6df6ecba944152ac84f1f9349 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T140232705899Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855682048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110741139456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110741139456, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110741139456, + "device": 4, + "inactive_split_bytes_current": 6536249344, + "inactive_split_bytes_peak": 8590935040, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:32.705503Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855682048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110741139456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110741139456, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110741139456, + "device": 4, + "inactive_split_bytes_current": 6532055040, + "inactive_split_bytes_peak": 8590935040, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:31.676509Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6536249344, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8590935040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112621256704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112621256704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T140232705899Z.pickle", + "timestamp_utc": "2026-08-01T14:02:32.809049Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T140232705899Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T140232705899Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..acdec32978b0687d8f204b8f6914e66871ecd151 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T140232705899Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8931415b4cb5c77282f19ba67b9c87d47455e2512df1b4533b6b01c3dd2774e +size 5621497 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T140233851343Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T140233851343Z.json new file mode 100644 index 0000000000000000000000000000000000000000..176ceab17bf15b80c8544af3334147abc774d8db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T140233851343Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855682048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110741139456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110741139456, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110741139456, + "device": 4, + "inactive_split_bytes_current": 7180075008, + "inactive_split_bytes_peak": 8590935040, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:33.850929Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855682048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110741139456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110741139456, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110741139456, + "device": 4, + "inactive_split_bytes_current": 6536249344, + "inactive_split_bytes_peak": 8590935040, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:32.823634Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7180075008, + "inactive_split_bytes_current_delta": 643825664, + "inactive_split_bytes_peak": 8590935040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112621256704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112621256704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T140233851343Z.pickle", + "timestamp_utc": "2026-08-01T14:02:33.954720Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T140233851343Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T140233851343Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e57252cdece4db23e28c41c94b766b8a6eb04ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T140233851343Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94d4c33b4bb20d17797d673b1d7ae1ff49157bfd63536bc32aec8f0597410327 +size 5621225 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T140235008245Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T140235008245Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d015e751fd838ece827fcd842a81b5d21fd13b98 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T140235008245Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855682048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110741139456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110741139456, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110741139456, + "device": 4, + "inactive_split_bytes_current": 6467043328, + "inactive_split_bytes_peak": 8590935040, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:35.007840Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855682048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110741139456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110741139456, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110741139456, + "device": 4, + "inactive_split_bytes_current": 7180075008, + "inactive_split_bytes_peak": 8590935040, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:33.969409Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6467043328, + "inactive_split_bytes_current_delta": -713031680, + "inactive_split_bytes_peak": 8590935040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112621256704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112621256704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T140235008245Z.pickle", + "timestamp_utc": "2026-08-01T14:02:35.117279Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T140235008245Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T140235008245Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a37588dc55d9563a10e2267f900762469b3f31d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T140235008245Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c563f60b77381dbc96f24f0e7157bb3d905e04a7a0debb843a29f0418d38d832 +size 5620528 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T140236157678Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T140236157678Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3ecbf8e7c219ba6a3cbd443a7604332e74d98dde --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T140236157678Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855682048, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110741139456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110741139456, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110741139456, + "device": 4, + "inactive_split_bytes_current": 7115063296, + "inactive_split_bytes_peak": 8590935040, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:36.157280Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855682048, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110741139456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110741139456, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110741139456, + "device": 4, + "inactive_split_bytes_current": 6467043328, + "inactive_split_bytes_peak": 8590935040, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:35.134929Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7115063296, + "inactive_split_bytes_current_delta": 648019968, + "inactive_split_bytes_peak": 8590935040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112621256704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112621256704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T140236157678Z.pickle", + "timestamp_utc": "2026-08-01T14:02:36.259571Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T140236157678Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T140236157678Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e9b3decfa5ba25d3db9c509a4fa370df5da178b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T140236157678Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c6d2c41717252d09e26cc8f553746450ac3f8440fc8885477cc3d9c2089d055 +size 5620558 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T140334988996Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T140334988996Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9480e83ae34dc887914c2761a7d49958cfd1408a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T140334988996Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66805039104, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110741139456, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66805039104, + "active_bytes_peak": 110741139456, + "allocated_bytes_current": 66805039104, + "allocated_bytes_peak": 110741139456, + "device": 4, + "inactive_split_bytes_current": 6496714752, + "inactive_split_bytes_peak": 8590935040, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:03:34.988516Z", + "world_size": 8 + }, + "allocated_bytes_current": 66805039104, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110741139456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855682048, + "active_bytes_peak": 110741139456, + "allocated_bytes_current": 66855682048, + "allocated_bytes_peak": 110741139456, + "device": 4, + "inactive_split_bytes_current": 7115063296, + "inactive_split_bytes_peak": 8590935040, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112621256704, + "reserved_bytes_peak": 112621256704, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:36.274046Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6496714752, + "inactive_split_bytes_current_delta": -618348544, + "inactive_split_bytes_peak": 8590935040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112621256704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112621256704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T140334988996Z.pickle", + "timestamp_utc": "2026-08-01T14:03:35.198334Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T140334988996Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T140334988996Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..306793cf00934108149247baa153c9a7c3c2e77a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T140334988996Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9efc90b7beb6b8ac09bd995aae85aa371e6b1e184a1dcbbc81f4027b7c907693 +size 10192540 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T140226665209Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T140226665209Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fabb4064fea5a4ed5d8e3daa72239c888e03f3e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T140226665209Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855515136, + "active_bytes_current_delta": 31624059904, + "active_bytes_peak": 87100909056, + "active_bytes_peak_delta": 51634572800, + "after": { + "active_bytes_current": 66855515136, + "active_bytes_peak": 87100909056, + "allocated_bytes_current": 66855515136, + "allocated_bytes_peak": 87100909056, + "device": 5, + "inactive_split_bytes_current": 6460918784, + "inactive_split_bytes_peak": 8099513344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 93778345984, + "reserved_bytes_peak": 93778345984, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T14:02:26.664626Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855515136, + "allocated_bytes_current_delta": 31624059904, + "allocated_bytes_peak": 87100909056, + "allocated_bytes_peak_delta": 51634572800, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:01:19.780641Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6460918784, + "inactive_split_bytes_current_delta": 6328099840, + "inactive_split_bytes_peak": 8099513344, + "inactive_split_bytes_peak_delta": 7939127296, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 93778345984, + "reserved_bytes_current_delta": 54534340608, + "reserved_bytes_peak": 93778345984, + "reserved_bytes_peak_delta": 54534340608, + "segment_count_current": 515, + "segment_count_current_delta": 444, + "segment_count_peak": 515, + "segment_count_peak_delta": 444, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T140226665209Z.pickle", + "timestamp_utc": "2026-08-01T14:02:26.776711Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T140226665209Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T140226665209Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e2e59f867a6dc3c357136cbed93b414675b762cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T140226665209Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e6f7788c20bed286a5cfda023f8d0f6388a444623def35d219043a7a4a3b9b0 +size 5388894 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T140227845668Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T140227845668Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7ddd4cacbb00bf08932a0658fe11aa796eb77d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T140227845668Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 110730922496, + "active_bytes_peak_delta": 23630013440, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110730922496, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110730922496, + "device": 5, + "inactive_split_bytes_current": 7112084480, + "inactive_split_bytes_peak": 8099513344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112856137728, + "reserved_bytes_peak": 112856137728, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:27.845257Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 110730922496, + "allocated_bytes_peak_delta": 23630013440, + "before": { + "active_bytes_current": 66855515136, + "active_bytes_peak": 87100909056, + "allocated_bytes_current": 66855515136, + "allocated_bytes_peak": 87100909056, + "device": 5, + "inactive_split_bytes_current": 6460918784, + "inactive_split_bytes_peak": 8099513344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 93778345984, + "reserved_bytes_peak": 93778345984, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T14:02:26.795938Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7112084480, + "inactive_split_bytes_current_delta": 651165696, + "inactive_split_bytes_peak": 8099513344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112856137728, + "reserved_bytes_current_delta": 19077791744, + "reserved_bytes_peak": 112856137728, + "reserved_bytes_peak_delta": 19077791744, + "segment_count_current": 596, + "segment_count_current_delta": 81, + "segment_count_peak": 596, + "segment_count_peak_delta": 81, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T140227845668Z.pickle", + "timestamp_utc": "2026-08-01T14:02:27.956572Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T140227845668Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T140227845668Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a47033d323cd429a9200456c11ba3abfa6ab1aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T140227845668Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2af2d6401e8a3b327eb3a8eab1ea9239368ad3c907c2260e0cd8c49202254fa +size 5618403 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T140228981496Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T140228981496Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7d3b59e4942d3d60954bac80bcc8f68cb5c02e56 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T140228981496Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734086144, + "active_bytes_peak_delta": 3163648, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 5, + "inactive_split_bytes_current": 6883494912, + "inactive_split_bytes_peak": 8151070208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112856137728, + "reserved_bytes_peak": 112856137728, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:28.981092Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734086144, + "allocated_bytes_peak_delta": 3163648, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110730922496, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110730922496, + "device": 5, + "inactive_split_bytes_current": 7112084480, + "inactive_split_bytes_peak": 8099513344, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112856137728, + "reserved_bytes_peak": 112856137728, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:27.971667Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6883494912, + "inactive_split_bytes_current_delta": -228589568, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 51556864, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112856137728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112856137728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T140228981496Z.pickle", + "timestamp_utc": "2026-08-01T14:02:29.090697Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T140228981496Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T140228981496Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..74ec8a1c007a8ce43c3a1a2c7d390cd5a8e5fe59 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T140228981496Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0591b43fd1e4e1fed2b8fabf41044b05d14f87ec2222a8b09af86cbcec2dc62d +size 5620959 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T140230132297Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T140230132297Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f0e76e13dc54db0e8212b3b575983d55ef8615b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T140230132297Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734086144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 5, + "inactive_split_bytes_current": 7118375936, + "inactive_split_bytes_peak": 8151070208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112856137728, + "reserved_bytes_peak": 112856137728, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:30.131895Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734086144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 5, + "inactive_split_bytes_current": 6883494912, + "inactive_split_bytes_peak": 8151070208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112856137728, + "reserved_bytes_peak": 112856137728, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:29.105794Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7118375936, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112856137728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112856137728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T140230132297Z.pickle", + "timestamp_utc": "2026-08-01T14:02:30.240045Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T140230132297Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T140230132297Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ceb3c5802946f2ed999cf6e7ed7489802eda51d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T140230132297Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:075733e9debebb9abe2d8ae2fa6f338f8e01e71e2a89f6b2af97313786f35a86 +size 5620455 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T140231554675Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T140231554675Z.json new file mode 100644 index 0000000000000000000000000000000000000000..58c3a9d2c51be228155dbd31514d2386d0f3a1d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T140231554675Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734086144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 5, + "inactive_split_bytes_current": 6474550272, + "inactive_split_bytes_peak": 8151070208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112856137728, + "reserved_bytes_peak": 112856137728, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:31.554253Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734086144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 5, + "inactive_split_bytes_current": 7118375936, + "inactive_split_bytes_peak": 8151070208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112856137728, + "reserved_bytes_peak": 112856137728, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:30.254811Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6474550272, + "inactive_split_bytes_current_delta": -643825664, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112856137728, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112856137728, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T140231554675Z.pickle", + "timestamp_utc": "2026-08-01T14:02:31.663652Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T140231554675Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T140231554675Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ef6970507e89b734bbf1216dcdc8d6fd475d8f19 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T140231554675Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2392e6ff56598a767f546345da84774f09ee8a5b7107704c8945471722830538 +size 5620025 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T140232705937Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T140232705937Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db086c8139d30664cff70dd48a437ce74b2fbc96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T140232705937Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 4763136, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 5, + "inactive_split_bytes_current": 7298731008, + "inactive_split_bytes_peak": 8151070208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:32.705521Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 4763136, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110734086144, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110734086144, + "device": 5, + "inactive_split_bytes_current": 6474550272, + "inactive_split_bytes_peak": 8151070208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112856137728, + "reserved_bytes_peak": 112856137728, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:31.678289Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7298731008, + "inactive_split_bytes_current_delta": 824180736, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 597, + "segment_count_current_delta": 1, + "segment_count_peak": 597, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T140232705937Z.pickle", + "timestamp_utc": "2026-08-01T14:02:32.808387Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T140232705937Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T140232705937Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2daba8b98704729cee8c644c82074c4558d13193 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T140232705937Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4d0cdaa39b0ea0423c0524f7c1063b5b4991a0f209c49ee480cf15a7580c306 +size 5618279 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T140233851340Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T140233851340Z.json new file mode 100644 index 0000000000000000000000000000000000000000..85682904a0fac3a7bd40f39ee64ec5edfabf1076 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T140233851340Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 5, + "inactive_split_bytes_current": 6583602176, + "inactive_split_bytes_peak": 8151070208, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:33.850933Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 5, + "inactive_split_bytes_current": 7298731008, + "inactive_split_bytes_peak": 8151070208, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:32.823135Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6583602176, + "inactive_split_bytes_current_delta": -715128832, + "inactive_split_bytes_peak": 8151070208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T140233851340Z.pickle", + "timestamp_utc": "2026-08-01T14:02:33.960300Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T140233851340Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T140233851340Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..92ae986b880789433da315ea17908701f94f3abe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T140233851340Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb71fd6fac8d94c22ca11fb2edc98d0c38faa1afd6bd5b13df474a4d285468ca +size 5619580 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T140235008189Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T140235008189Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f93be130c869ebf554229022319add282e15884a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T140235008189Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 5, + "inactive_split_bytes_current": 8009665536, + "inactive_split_bytes_peak": 8207998976, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:35.007778Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 5, + "inactive_split_bytes_current": 6583602176, + "inactive_split_bytes_peak": 8151070208, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:33.977611Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8009665536, + "inactive_split_bytes_current_delta": 1426063360, + "inactive_split_bytes_peak": 8207998976, + "inactive_split_bytes_peak_delta": 56928768, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T140235008189Z.pickle", + "timestamp_utc": "2026-08-01T14:02:35.108387Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T140235008189Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T140235008189Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7cf60834c1098cf4ea787e771309d4da865dd8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T140235008189Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:897155306ebba910fd880469874ae90bc0fc3356865453a667513e273c7e1527 +size 5619869 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T140236157677Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T140236157677Z.json new file mode 100644 index 0000000000000000000000000000000000000000..02c5db4d384c0ecd3ea68c85c115bfc38ae79d87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T140236157677Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 5, + "inactive_split_bytes_current": 7474891776, + "inactive_split_bytes_peak": 8516075520, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:36.157274Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 5, + "inactive_split_bytes_current": 8009665536, + "inactive_split_bytes_peak": 8207998976, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:35.122625Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7474891776, + "inactive_split_bytes_current_delta": -534773760, + "inactive_split_bytes_peak": 8516075520, + "inactive_split_bytes_peak_delta": 308076544, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T140236157677Z.pickle", + "timestamp_utc": "2026-08-01T14:02:36.259490Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T140236157677Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T140236157677Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b435d149c14214fa0d651d708953f848a639c936 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T140236157677Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d04b69df99a016b5a14b30c5d480f9b5b31dde808ee91688a2aefb55b3f2d5 +size 5616680 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T140334988996Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T140334988996Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80464fc8b0403b8303f29a47caca347f03f9c178 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T140334988996Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803823616, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110738849280, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803823616, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66803823616, + "allocated_bytes_peak": 110738849280, + "device": 5, + "inactive_split_bytes_current": 6466472960, + "inactive_split_bytes_peak": 8516075520, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:03:34.988527Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803823616, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110738849280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110738849280, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110738849280, + "device": 5, + "inactive_split_bytes_current": 7474891776, + "inactive_split_bytes_peak": 8516075520, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112858234880, + "reserved_bytes_peak": 112858234880, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:36.274082Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6466472960, + "inactive_split_bytes_current_delta": -1008418816, + "inactive_split_bytes_peak": 8516075520, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112858234880, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112858234880, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T140334988996Z.pickle", + "timestamp_utc": "2026-08-01T14:03:35.192547Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T140334988996Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T140334988996Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a163b9c68b2107a06e87586bc375dee2ab7ea0b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T140334988996Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:572437aaad38b0184117ba85c52a24194624f8172928a9af894881ce4a97664a +size 9363227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T140226665217Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T140226665217Z.json new file mode 100644 index 0000000000000000000000000000000000000000..52c4fd15c532ddf0ec8f77799ac6fb88a4ca8b8d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T140226665217Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855515136, + "active_bytes_current_delta": 31624059904, + "active_bytes_peak": 87100909056, + "active_bytes_peak_delta": 51634572800, + "after": { + "active_bytes_current": 66855515136, + "active_bytes_peak": 87100909056, + "allocated_bytes_current": 66855515136, + "allocated_bytes_peak": 87100909056, + "device": 6, + "inactive_split_bytes_current": 6460918784, + "inactive_split_bytes_peak": 8026694656, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 93778345984, + "reserved_bytes_peak": 93778345984, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T14:02:26.664626Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855515136, + "allocated_bytes_current_delta": 31624059904, + "allocated_bytes_peak": 87100909056, + "allocated_bytes_peak_delta": 51634572800, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:01:19.780594Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6460918784, + "inactive_split_bytes_current_delta": 6328099840, + "inactive_split_bytes_peak": 8026694656, + "inactive_split_bytes_peak_delta": 7866308608, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 93778345984, + "reserved_bytes_current_delta": 54534340608, + "reserved_bytes_peak": 93778345984, + "reserved_bytes_peak_delta": 54534340608, + "segment_count_current": 515, + "segment_count_current_delta": 444, + "segment_count_peak": 515, + "segment_count_peak_delta": 444, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T140226665217Z.pickle", + "timestamp_utc": "2026-08-01T14:02:26.774828Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T140226665217Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T140226665217Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bfe15b554d8ad3cb18ecf4ede221270d79b49075 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T140226665217Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a874c68ed7907bb98259f9c8fd971b00a653b040999669fa4f8de1855bfa480 +size 5388533 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T140227845725Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T140227845725Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5995c1b1feabfbc6f4f7ffb5313b55dbe9ad654e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T140227845725Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 110728398848, + "active_bytes_peak_delta": 23627489792, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110728398848, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110728398848, + "device": 6, + "inactive_split_bytes_current": 6826871808, + "inactive_split_bytes_peak": 8262656000, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:27.845325Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 110728398848, + "allocated_bytes_peak_delta": 23627489792, + "before": { + "active_bytes_current": 66855515136, + "active_bytes_peak": 87100909056, + "allocated_bytes_current": 66855515136, + "allocated_bytes_peak": 87100909056, + "device": 6, + "inactive_split_bytes_current": 6460918784, + "inactive_split_bytes_peak": 8026694656, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 93778345984, + "reserved_bytes_peak": 93778345984, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T14:02:26.790734Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6826871808, + "inactive_split_bytes_current_delta": 365953024, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 235961344, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 18962448384, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 18962448384, + "segment_count_current": 596, + "segment_count_current_delta": 81, + "segment_count_peak": 596, + "segment_count_peak_delta": 81, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T140227845725Z.pickle", + "timestamp_utc": "2026-08-01T14:02:27.953346Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T140227845725Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T140227845725Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ecde5ffc37d3db06792bd56fb6ee9c56c7ecb8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T140227845725Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a715453b185badb8f39d06ece48a73bca58b065f00d858315c47ecf89efc8062 +size 5618448 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T140228981454Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T140228981454Z.json new file mode 100644 index 0000000000000000000000000000000000000000..60e9eb72afee271406a4f369de12167c12a49c87 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T140228981454Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110732865536, + "active_bytes_peak_delta": 4466688, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 6, + "inactive_split_bytes_current": 7411977216, + "inactive_split_bytes_peak": 8262656000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:28.981060Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110732865536, + "allocated_bytes_peak_delta": 4466688, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110728398848, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110728398848, + "device": 6, + "inactive_split_bytes_current": 6826871808, + "inactive_split_bytes_peak": 8262656000, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:27.965158Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7411977216, + "inactive_split_bytes_current_delta": 585105408, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T140228981454Z.pickle", + "timestamp_utc": "2026-08-01T14:02:29.076775Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T140228981454Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T140228981454Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..518e9de82d11c27c393db75a1b0c50ada4db9cf1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T140228981454Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ace0eaaf2f900179cc6709e08d8eb85ea9d267509a599b3986dfc59ef2ce801 +size 5618329 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T140230132313Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T140230132313Z.json new file mode 100644 index 0000000000000000000000000000000000000000..05f55ecd5c9cd5c54be40c0ec6145772ae3c7d7e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T140230132313Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110732865536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 6, + "inactive_split_bytes_current": 7583943680, + "inactive_split_bytes_peak": 8262656000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:30.131903Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110732865536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 6, + "inactive_split_bytes_current": 7411977216, + "inactive_split_bytes_peak": 8262656000, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:29.087969Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7583943680, + "inactive_split_bytes_current_delta": 171966464, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T140230132313Z.pickle", + "timestamp_utc": "2026-08-01T14:02:30.234070Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T140230132313Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T140230132313Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1290b7d4c79ce50f510992dd3686af48507650f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T140230132313Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbd0e8440969680cf7554063ae7785e6e9b998b5c5d4fae4b6a57722a0169352 +size 5619191 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T140231554676Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T140231554676Z.json new file mode 100644 index 0000000000000000000000000000000000000000..554a246373f40ddf99f7f78b56a674cd0dfb44cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T140231554676Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110732865536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 6, + "inactive_split_bytes_current": 7122570240, + "inactive_split_bytes_peak": 8262656000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:31.554256Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110732865536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 6, + "inactive_split_bytes_current": 7583943680, + "inactive_split_bytes_peak": 8262656000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:30.245649Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7122570240, + "inactive_split_bytes_current_delta": -461373440, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T140231554676Z.pickle", + "timestamp_utc": "2026-08-01T14:02:31.657094Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T140231554676Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T140231554676Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b10c47d4477882eb65b28c351128584730c57600 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T140231554676Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e263720963c34f3fd381f1d8d98afdeaaa934d791f41c2768145ab65b08dc5b4 +size 5619710 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T140232705915Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T140232705915Z.json new file mode 100644 index 0000000000000000000000000000000000000000..62a6ba539b193922acef39cf610617bd1b1110c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T140232705915Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110732865536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 6, + "inactive_split_bytes_current": 7653149696, + "inactive_split_bytes_peak": 8262656000, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:32.705520Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110732865536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 6, + "inactive_split_bytes_current": 7122570240, + "inactive_split_bytes_peak": 8262656000, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:31.669192Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7653149696, + "inactive_split_bytes_current_delta": 530579456, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T140232705915Z.pickle", + "timestamp_utc": "2026-08-01T14:02:32.798344Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T140232705915Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T140232705915Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b1bedd101f7f77ef8cf1bdc7aa82d0bdfee0c0da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T140232705915Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0076ecaa603ee208e4b274f118e92eb4717274cccc59216063374af7ab902dc6 +size 5620023 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T140233851437Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T140233851437Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1b8aa9ecc3034842c7d0331a2871c6809a22c3a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T140233851437Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110732865536, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 6, + "inactive_split_bytes_current": 7648955392, + "inactive_split_bytes_peak": 8262656000, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:33.851013Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110732865536, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 6, + "inactive_split_bytes_current": 7653149696, + "inactive_split_bytes_peak": 8262656000, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:32.809195Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7648955392, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T140233851437Z.pickle", + "timestamp_utc": "2026-08-01T14:02:33.945743Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T140233851437Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T140233851437Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ef17f0c043c0b4ee24a9230eae8e25d4f3b0bb8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T140233851437Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b69cc1ca39f3d6feaf1fee5bfbfbf0932b520c192f2239390e066e53bb8ccf25 +size 5619371 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T140235008215Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T140235008215Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c10754267beee14534892ee0839eaedd2d063925 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T140235008215Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110737715712, + "active_bytes_peak_delta": 4850176, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110737715712, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110737715712, + "device": 6, + "inactive_split_bytes_current": 7590235136, + "inactive_split_bytes_peak": 8262656000, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:35.007794Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110737715712, + "allocated_bytes_peak_delta": 4850176, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110732865536, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110732865536, + "device": 6, + "inactive_split_bytes_current": 7648955392, + "inactive_split_bytes_peak": 8262656000, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:33.956733Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7590235136, + "inactive_split_bytes_current_delta": -58720256, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T140235008215Z.pickle", + "timestamp_utc": "2026-08-01T14:02:35.110029Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T140235008215Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T140235008215Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d0044322eaa6877b2f292750decab63a80403704 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T140235008215Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79e9f61240ff318643be6b4b565d82c4dbdbb2aee3bf353ad52fe648c0e4ac65 +size 5619393 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T140236157813Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T140236157813Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6d973c606085dcab7835201ee3b4cb7c94753fe6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T140236157813Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854466560, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110737715712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110737715712, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110737715712, + "device": 6, + "inactive_split_bytes_current": 7177096192, + "inactive_split_bytes_peak": 8262656000, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:36.157419Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854466560, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110737715712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110737715712, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110737715712, + "device": 6, + "inactive_split_bytes_current": 7590235136, + "inactive_split_bytes_peak": 8262656000, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:35.123303Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7177096192, + "inactive_split_bytes_current_delta": -413138944, + "inactive_split_bytes_peak": 8262656000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T140236157813Z.pickle", + "timestamp_utc": "2026-08-01T14:02:36.249267Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T140236157813Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T140236157813Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f03bdd6fde199bedd4db74ba1ae2c9401130ebd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T140236157813Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79f30bb8ac5ad71497d2177ef6216434f9aeecc7cd198d408e482146795b8e5d +size 5618389 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T140334988975Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T140334988975Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d594f2b9a3eb8443c41b09954529f59a605a353e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T140334988975Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803823616, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110737715712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803823616, + "active_bytes_peak": 110737715712, + "allocated_bytes_current": 66803823616, + "allocated_bytes_peak": 110737715712, + "device": 6, + "inactive_split_bytes_current": 6466472960, + "inactive_split_bytes_peak": 8499300352, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:03:34.988509Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803823616, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110737715712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854466560, + "active_bytes_peak": 110737715712, + "allocated_bytes_current": 66854466560, + "allocated_bytes_peak": 110737715712, + "device": 6, + "inactive_split_bytes_current": 7177096192, + "inactive_split_bytes_peak": 8262656000, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:36.259956Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6466472960, + "inactive_split_bytes_current_delta": -710623232, + "inactive_split_bytes_peak": 8499300352, + "inactive_split_bytes_peak_delta": 236644352, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 596, + "segment_count_current_delta": 0, + "segment_count_peak": 596, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T140334988975Z.pickle", + "timestamp_utc": "2026-08-01T14:03:35.182348Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T140334988975Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T140334988975Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0fe3199199b509910a59d999ff8a5dc2ffb51c7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T140334988975Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7e73a6e81505462a2fd03f1f7d6662b3086659d89276832813df100fa4a325c +size 9526373 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T140226665087Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T140226665087Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0c0c6c97a80273e74f8716a8c5c91b83992998d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T140226665087Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855515648, + "active_bytes_current_delta": 31624060416, + "active_bytes_peak": 87100909056, + "active_bytes_peak_delta": 51634572800, + "after": { + "active_bytes_current": 66855515648, + "active_bytes_peak": 87100909056, + "allocated_bytes_current": 66855515648, + "allocated_bytes_peak": 87100909056, + "device": 7, + "inactive_split_bytes_current": 6345574912, + "inactive_split_bytes_peak": 8101610496, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 93543464960, + "reserved_bytes_peak": 93543464960, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T14:02:26.664612Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855515648, + "allocated_bytes_current_delta": 31624060416, + "allocated_bytes_peak": 87100909056, + "allocated_bytes_peak_delta": 51634572800, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:01:19.784519Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6345574912, + "inactive_split_bytes_current_delta": 6212755968, + "inactive_split_bytes_peak": 8101610496, + "inactive_split_bytes_peak_delta": 7941224448, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 93543464960, + "reserved_bytes_current_delta": 54299459584, + "reserved_bytes_peak": 93543464960, + "reserved_bytes_peak_delta": 54299459584, + "segment_count_current": 515, + "segment_count_current_delta": 444, + "segment_count_peak": 515, + "segment_count_peak_delta": 444, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T140226665087Z.pickle", + "timestamp_utc": "2026-08-01T14:02:26.773844Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T140226665087Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T140226665087Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3c530b739314e09c0ca3884c047ddf25464665c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T140226665087Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a89add539dc881abb6bada0dd78980aac1e941cb2c54ee554381c70967b82f97 +size 5391591 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T140227845700Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T140227845700Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a7058e9c16e8758211058faf1c138d4238292457 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T140227845700Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 110728399360, + "active_bytes_peak_delta": 23627490304, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110728399360, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110728399360, + "device": 7, + "inactive_split_bytes_current": 6820579840, + "inactive_split_bytes_peak": 8260558336, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112738697216, + "reserved_bytes_peak": 112738697216, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:27.845295Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 110728399360, + "allocated_bytes_peak_delta": 23627490304, + "before": { + "active_bytes_current": 66855515648, + "active_bytes_peak": 87100909056, + "allocated_bytes_current": 66855515648, + "allocated_bytes_peak": 87100909056, + "device": 7, + "inactive_split_bytes_current": 6345574912, + "inactive_split_bytes_peak": 8101610496, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 93543464960, + "reserved_bytes_peak": 93543464960, + "segment_count_current": 515, + "segment_count_peak": 515, + "timestamp_utc": "2026-08-01T14:02:26.791336Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6820579840, + "inactive_split_bytes_current_delta": 475004928, + "inactive_split_bytes_peak": 8260558336, + "inactive_split_bytes_peak_delta": 158947840, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112738697216, + "reserved_bytes_current_delta": 19195232256, + "reserved_bytes_peak": 112738697216, + "reserved_bytes_peak_delta": 19195232256, + "segment_count_current": 596, + "segment_count_current_delta": 81, + "segment_count_peak": 596, + "segment_count_peak_delta": 81, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T140227845700Z.pickle", + "timestamp_utc": "2026-08-01T14:02:27.954415Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T140227845700Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T140227845700Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d9f26991c86a03f8dd50fdd630fc4f5b5c90d683 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T140227845700Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58174ca192cb936cbe9a70412debbe2f284cc05612234343e67e99d471341a9f +size 5621139 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T140228981516Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T140228981516Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3294f2e47f23339ea7cd3cd45be96509314540d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T140228981516Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734832128, + "active_bytes_peak_delta": 6432768, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110734832128, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110734832128, + "device": 7, + "inactive_split_bytes_current": 7055460864, + "inactive_split_bytes_peak": 8353447424, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:28.981113Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734832128, + "allocated_bytes_peak_delta": 6432768, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110728399360, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110728399360, + "device": 7, + "inactive_split_bytes_current": 6820579840, + "inactive_split_bytes_peak": 8260558336, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112738697216, + "reserved_bytes_peak": 112738697216, + "segment_count_current": 596, + "segment_count_peak": 596, + "timestamp_utc": "2026-08-01T14:02:27.967883Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7055460864, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 8353447424, + "inactive_split_bytes_peak_delta": 92889088, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 597, + "segment_count_current_delta": 1, + "segment_count_peak": 597, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T140228981516Z.pickle", + "timestamp_utc": "2026-08-01T14:02:29.083241Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T140228981516Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T140228981516Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b21f9f4df427873beb1efed000a1497e2c00a7c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T140228981516Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cf4a571b6b809c5bb775b34cb34d14caa2d0b60e6cb25f103804dff5ccf628a +size 5622727 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T140230132337Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T140230132337Z.json new file mode 100644 index 0000000000000000000000000000000000000000..13388c01565d729bae9c55ac84398c8e7d8b400b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T140230132337Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734832128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110734832128, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110734832128, + "device": 7, + "inactive_split_bytes_current": 7118375424, + "inactive_split_bytes_peak": 8353447424, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:30.131925Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734832128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110734832128, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110734832128, + "device": 7, + "inactive_split_bytes_current": 7055460864, + "inactive_split_bytes_peak": 8353447424, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:29.096375Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7118375424, + "inactive_split_bytes_current_delta": 62914560, + "inactive_split_bytes_peak": 8353447424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T140230132337Z.pickle", + "timestamp_utc": "2026-08-01T14:02:30.235605Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T140230132337Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T140230132337Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5bc3fd171242ef4431b1c7814deab5c034b863cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T140230132337Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6b66cd27d3a9748963504e696338936eee8947ce2d9f63d306428b8b81f03af +size 5619477 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T140231554678Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T140231554678Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cc97726cc4a086e891dff29c47fd8bf1bb1532c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T140231554678Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734832128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110734832128, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110734832128, + "device": 7, + "inactive_split_bytes_current": 7367936512, + "inactive_split_bytes_peak": 8353447424, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:31.554227Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734832128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110734832128, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110734832128, + "device": 7, + "inactive_split_bytes_current": 7118375424, + "inactive_split_bytes_peak": 8353447424, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:30.248903Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7367936512, + "inactive_split_bytes_current_delta": 249561088, + "inactive_split_bytes_peak": 8353447424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T140231554678Z.pickle", + "timestamp_utc": "2026-08-01T14:02:31.659091Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T140231554678Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T140231554678Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27a87fe68ca031654f2ae0c3ab9819f48eb9ef95 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T140231554678Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e52047e8ba2ee93c0e30b484ccdc11dc49cd8ebdecd0f63767ee8c86b90f3fe +size 5621927 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T140232705893Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T140232705893Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c427cb512468710ecd43c10a5685978993740c37 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T140232705893Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734832128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110734832128, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110734832128, + "device": 7, + "inactive_split_bytes_current": 7416171008, + "inactive_split_bytes_peak": 8353447424, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:32.705494Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734832128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110734832128, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110734832128, + "device": 7, + "inactive_split_bytes_current": 7367936512, + "inactive_split_bytes_peak": 8353447424, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:31.672091Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7416171008, + "inactive_split_bytes_current_delta": 48234496, + "inactive_split_bytes_peak": 8353447424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T140232705893Z.pickle", + "timestamp_utc": "2026-08-01T14:02:32.808984Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T140232705893Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T140232705893Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bc9156d6ee5b04d8014bb0767702f2465b9c4e3d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T140232705893Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb83085a4e94ff45eeea7fac08afdd305ada4e8581a5a96f570c7e6541e87614 +size 5621836 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T140233851405Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T140233851405Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7a709aef7c74aaf09b24ca252d68d2570511c913 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T140233851405Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734832128, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110734832128, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110734832128, + "device": 7, + "inactive_split_bytes_current": 7120472576, + "inactive_split_bytes_peak": 8624964608, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:33.850996Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734832128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110734832128, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110734832128, + "device": 7, + "inactive_split_bytes_current": 7416171008, + "inactive_split_bytes_peak": 8353447424, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:32.824149Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7120472576, + "inactive_split_bytes_current_delta": -295698432, + "inactive_split_bytes_peak": 8624964608, + "inactive_split_bytes_peak_delta": 271517184, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T140233851405Z.pickle", + "timestamp_utc": "2026-08-01T14:02:33.948418Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T140233851405Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T140233851405Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..094e779cf716865eabc2929ca45922040a22cdb4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T140233851405Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:446554dc9c74c5a3d75aa8a833910c160f4fa288b843f2826f6b83b297303575 +size 5620961 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T140235008189Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T140235008189Z.json new file mode 100644 index 0000000000000000000000000000000000000000..012a11b655012e08c49deccf0c82b9a3a1473ce9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T140235008189Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110737325568, + "active_bytes_peak_delta": 2493440, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110737325568, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110737325568, + "device": 7, + "inactive_split_bytes_current": 7942556160, + "inactive_split_bytes_peak": 8624964608, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:35.007771Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110737325568, + "allocated_bytes_peak_delta": 2493440, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110734832128, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110734832128, + "device": 7, + "inactive_split_bytes_current": 7120472576, + "inactive_split_bytes_peak": 8624964608, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:33.961105Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7942556160, + "inactive_split_bytes_current_delta": 822083584, + "inactive_split_bytes_peak": 8624964608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T140235008189Z.pickle", + "timestamp_utc": "2026-08-01T14:02:35.105924Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T140235008189Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T140235008189Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a26fd2fd9122b4bcbfa44941baa50b5745baf685 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T140235008189Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f52c70bd05d7a41ed55d91c95eea57c9fce3ebeb5253f4005e6161a1e3a6be9e +size 5621317 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T140236157679Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T140236157679Z.json new file mode 100644 index 0000000000000000000000000000000000000000..26c5eca6cc917aabe7182cb24f4763909de72521 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T140236157679Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854467072, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110737325568, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110737325568, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110737325568, + "device": 7, + "inactive_split_bytes_current": 7237913088, + "inactive_split_bytes_peak": 8624964608, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:36.157271Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854467072, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110737325568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110737325568, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110737325568, + "device": 7, + "inactive_split_bytes_current": 7942556160, + "inactive_split_bytes_peak": 8624964608, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:35.118376Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7237913088, + "inactive_split_bytes_current_delta": -704643072, + "inactive_split_bytes_peak": 8624964608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T140236157679Z.pickle", + "timestamp_utc": "2026-08-01T14:02:36.253806Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T140236157679Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T140236157679Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9b6eb0bffdecd9b15296996fbd24db8916d116e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T140236157679Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3b04d7b8ecb85e85325d1052a6c88e9ab517db03703e0cedc4edffbd752c030 +size 5621304 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T140334988996Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T140334988996Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f45d93dd2805ca8d5e212a1aac3798050c48fa1c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T140334988996Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803824128, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110737325568, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803824128, + "active_bytes_peak": 110737325568, + "allocated_bytes_current": 66803824128, + "allocated_bytes_peak": 110737325568, + "device": 7, + "inactive_split_bytes_current": 6466472448, + "inactive_split_bytes_peak": 8624964608, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:03:34.988532Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803824128, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110737325568, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854467072, + "active_bytes_peak": 110737325568, + "allocated_bytes_current": 66854467072, + "allocated_bytes_peak": 110737325568, + "device": 7, + "inactive_split_bytes_current": 7237913088, + "inactive_split_bytes_peak": 8624964608, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112740794368, + "reserved_bytes_peak": 112740794368, + "segment_count_current": 597, + "segment_count_peak": 597, + "timestamp_utc": "2026-08-01T14:02:36.266098Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6466472448, + "inactive_split_bytes_current_delta": -771440640, + "inactive_split_bytes_peak": 8624964608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112740794368, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112740794368, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 597, + "segment_count_current_delta": 0, + "segment_count_peak": 597, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T140334988996Z.pickle", + "timestamp_utc": "2026-08-01T14:03:35.185060Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T140334988996Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T140334988996Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7d47dddf7870e9e7f63b173b5e43e63a782830fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T140334988996Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec401f4021e3cd3ff1a9c8ea828d3ec5cf8cd2693a46ec6dcf93a8575e10fbff +size 10180567 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..2d6898dd2f51a6840d5c82c4ce9b603002788c48 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 4096 +capacity: 1.00 +expert_dispatch_overlap: off +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6037 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 226 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T140619005186Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T140619005186Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c1a94e1a1bba6dd2a7c86927ec2496d38e0118ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T140619005186Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440173568, + "active_bytes_current_delta": 37208718336, + "active_bytes_peak": 86995328512, + "active_bytes_peak_delta": 51528992256, + "after": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 86995328512, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 86995328512, + "device": 0, + "inactive_split_bytes_current": 5489994752, + "inactive_split_bytes_peak": 8599057408, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 96116670464, + "reserved_bytes_peak": 96116670464, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.004610Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440173568, + "allocated_bytes_current_delta": 37208718336, + "allocated_bytes_peak": 86995328512, + "allocated_bytes_peak_delta": 51528992256, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T14:05:10.964289Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5489994752, + "inactive_split_bytes_current_delta": 5355078656, + "inactive_split_bytes_peak": 8599057408, + "inactive_split_bytes_peak_delta": 8438425600, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 96116670464, + "reserved_bytes_current_delta": 56870567936, + "reserved_bytes_peak": 96116670464, + "reserved_bytes_peak_delta": 56870567936, + "segment_count_current": 458, + "segment_count_current_delta": 386, + "segment_count_peak": 458, + "segment_count_peak_delta": 386, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T140619005186Z.pickle", + "timestamp_utc": "2026-08-01T14:06:19.121006Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T140619005186Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T140619005186Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1aed7bd2f44210f5a410fbed9454b9f42a9512ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T140619005186Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fccf5bda90c9cfe54966d1510e58cd309bf43749b6067543186b243b37fea3ee +size 5950972 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T140620358454Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T140620358454Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0fc47881ce4e6eb1e231840033d3731c7af0d1dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T140620358454Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440173568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110623067136, + "active_bytes_peak_delta": 23627738624, + "after": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110623067136, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110623067136, + "device": 0, + "inactive_split_bytes_current": 6133820416, + "inactive_split_bytes_peak": 8599057408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:20.358042Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440173568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110623067136, + "allocated_bytes_peak_delta": 23627738624, + "before": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 86995328512, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 86995328512, + "device": 0, + "inactive_split_bytes_current": 5489994752, + "inactive_split_bytes_peak": 8599057408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 96116670464, + "reserved_bytes_peak": 96116670464, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.138017Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6133820416, + "inactive_split_bytes_current_delta": 643825664, + "inactive_split_bytes_peak": 8599057408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 16856907776, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 16856907776, + "segment_count_current": 539, + "segment_count_current_delta": 81, + "segment_count_peak": 539, + "segment_count_peak_delta": 81, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T140620358454Z.pickle", + "timestamp_utc": "2026-08-01T14:06:20.472905Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T140620358454Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T140620358454Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1369423781b7f72c530dd998c9b3014fe2d08973 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T140620358454Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c146f50083f021aea702656180d493cdfb39409c8b19919c7738cbfd548d9fe7 +size 6651555 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T140621629506Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T140621629506Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d8e2880f498bd12a612ce1ab072e48551dcaa8c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T140621629506Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440173568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627678720, + "active_bytes_peak_delta": 4611584, + "after": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110627678720, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110627678720, + "device": 0, + "inactive_split_bytes_current": 6194637824, + "inactive_split_bytes_peak": 8599057408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:21.629113Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440173568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627678720, + "allocated_bytes_peak_delta": 4611584, + "before": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110623067136, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110623067136, + "device": 0, + "inactive_split_bytes_current": 6133820416, + "inactive_split_bytes_peak": 8599057408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:20.486970Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6194637824, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 8599057408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 539, + "segment_count_current_delta": 0, + "segment_count_peak": 539, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T140621629506Z.pickle", + "timestamp_utc": "2026-08-01T14:06:21.735868Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T140621629506Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T140621629506Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aedc8e51e8d98fb3439d73fb9c244b80c0f6b2a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T140621629506Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5db8ce01ce3e711d039405cdd16f8b6c2f9d6e959ce09364b2965be13f578b8a +size 6650851 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T140622919405Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T140622919405Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e11965ea5e8d907fe12e52ca6c3acb965fef8f92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T140622919405Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440173568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110628585472, + "active_bytes_peak_delta": 906752, + "after": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 7253699584, + "inactive_split_bytes_peak": 8599057408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:22.919001Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440173568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110628585472, + "allocated_bytes_peak_delta": 906752, + "before": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110627678720, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110627678720, + "device": 0, + "inactive_split_bytes_current": 6194637824, + "inactive_split_bytes_peak": 8599057408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:21.749293Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7253699584, + "inactive_split_bytes_current_delta": 1059061760, + "inactive_split_bytes_peak": 8599057408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 539, + "segment_count_current_delta": 0, + "segment_count_peak": 539, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T140622919405Z.pickle", + "timestamp_utc": "2026-08-01T14:06:23.024128Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T140622919405Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T140622919405Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6fd6d9dd686c8e48e8c97fe608cdc8cf182763a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T140622919405Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29c8a4a618d00558bab5fd16d7eb72f7a0609c682830936cd3b9357575e226fd +size 6650062 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T140624472448Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T140624472448Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a25868d585362c0906116ff215948ad116c0825 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T140624472448Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440173568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110628585472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 6077197312, + "inactive_split_bytes_peak": 8599057408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:24.472036Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440173568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110628585472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 7253699584, + "inactive_split_bytes_peak": 8599057408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:23.037469Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6077197312, + "inactive_split_bytes_current_delta": -1176502272, + "inactive_split_bytes_peak": 8599057408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 539, + "segment_count_current_delta": 0, + "segment_count_peak": 539, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T140624472448Z.pickle", + "timestamp_utc": "2026-08-01T14:06:24.585653Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T140624472448Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T140624472448Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a42612cf592e437b0875b4fab5256989c0b7bc0a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T140624472448Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:792d9518570acded30bf9419b912f3af424d3b14972c2f87d3d4697661c66917 +size 6649263 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T140625766268Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T140625766268Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d457e7efbabe6d494bc5278c8f48f1b9534ffedd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T140625766268Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440173568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110628585472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 6370798592, + "inactive_split_bytes_peak": 8599057408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:25.765838Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440173568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110628585472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 6077197312, + "inactive_split_bytes_peak": 8599057408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:24.604986Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6370798592, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 8599057408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 539, + "segment_count_current_delta": 0, + "segment_count_peak": 539, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T140625766268Z.pickle", + "timestamp_utc": "2026-08-01T14:06:25.872837Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T140625766268Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T140625766268Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..770581485a69dfe0f64cb472122b69e8bf78c600 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T140625766268Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ba2d68f27d5dde4acfd1f1be0e18faa3fdfde6a6cfc5cb046afdc43a69dda3b +size 6649200 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T140627053227Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T140627053227Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f2bdf3e1bb41cbbdddbb2b4215b0568f97caa97 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T140627053227Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440173568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110628585472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 6077197312, + "inactive_split_bytes_peak": 8599057408, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:27.052806Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440173568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110628585472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 6370798592, + "inactive_split_bytes_peak": 8599057408, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:25.886629Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6077197312, + "inactive_split_bytes_current_delta": -293601280, + "inactive_split_bytes_peak": 8599057408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 539, + "segment_count_current_delta": 0, + "segment_count_peak": 539, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T140627053227Z.pickle", + "timestamp_utc": "2026-08-01T14:06:27.160152Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T140627053227Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T140627053227Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e20552385b8e37f7b1009b1ad33abb8d3dc913f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T140627053227Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17d267f83d94e9cc01ff1cad5f34ff2fd087bca0219dfa356c659cb622ca4412 +size 6649157 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T140628495887Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T140628495887Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3fe180f5903522c2b2ed8d63e0c0cd830d408a74 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T140628495887Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440173568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110628585472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 6370798592, + "inactive_split_bytes_peak": 8599057408, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:28.495487Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440173568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110628585472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 6077197312, + "inactive_split_bytes_peak": 8599057408, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:27.173878Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6370798592, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 8599057408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 539, + "segment_count_current_delta": 0, + "segment_count_peak": 539, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T140628495887Z.pickle", + "timestamp_utc": "2026-08-01T14:06:28.601351Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T140628495887Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T140628495887Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d8e29937f39e205bcbeb601ee7f4a88734f4539 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T140628495887Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aa38bf25f5bc5b8113aec3bb2f21ce5c796590464b392a96dc79e729955af65 +size 6649213 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T140629778704Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T140629778704Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f58590bc8292ce5db16e2042ebc7dbd689fc60bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T140629778704Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440173568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110628585472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 6077197312, + "inactive_split_bytes_peak": 8599057408, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:29.778293Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440173568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110628585472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 6370798592, + "inactive_split_bytes_peak": 8599057408, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:28.614890Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6077197312, + "inactive_split_bytes_current_delta": -293601280, + "inactive_split_bytes_peak": 8599057408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 539, + "segment_count_current_delta": 0, + "segment_count_peak": 539, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T140629778704Z.pickle", + "timestamp_utc": "2026-08-01T14:06:29.883342Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T140629778704Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T140629778704Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0b544487c2982aee812702ab90a42dc253e5fac0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T140629778704Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a51656651b12f63dc27be2fcd8028f80f922bf54a3e6ca50cd882db3bd2f4153 +size 6649147 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T140722684079Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T140722684079Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ce34410a86903645c4ed4d111b433c54b2e2f0ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T140722684079Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165884416, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 110628585472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165884416, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 61165884416, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 5955562496, + "inactive_split_bytes_peak": 8599057408, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:07:22.683622Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165884416, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110628585472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440173568, + "active_bytes_peak": 110628585472, + "allocated_bytes_current": 72440173568, + "allocated_bytes_peak": 110628585472, + "device": 0, + "inactive_split_bytes_current": 6077197312, + "inactive_split_bytes_peak": 8599057408, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_peak": 112973578240, + "segment_count_current": 539, + "segment_count_peak": 539, + "timestamp_utc": "2026-08-01T14:06:29.897005Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5955562496, + "inactive_split_bytes_current_delta": -121634816, + "inactive_split_bytes_peak": 8599057408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 112973578240, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112973578240, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 539, + "segment_count_current_delta": 0, + "segment_count_peak": 539, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T140722684079Z.pickle", + "timestamp_utc": "2026-08-01T14:07:22.884770Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T140722684079Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T140722684079Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8455abe5ddc902b462dd788c0735d9cf5a50fbe5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T140722684079Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fe5b2d3a6e0c1bb81bb551082d49d3a721b4c44fff4061ef549824fcaf4a088 +size 10535774 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T140619005110Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T140619005110Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31def57f78bdbdfb7b7b98e129b3142c08fd6239 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T140619005110Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 87003564032, + "active_bytes_peak_delta": 51537227776, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87003564032, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 87003564032, + "device": 1, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_peak": 9402926080, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 96114573312, + "reserved_bytes_peak": 96114573312, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.004526Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 87003564032, + "allocated_bytes_peak_delta": 51537227776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:05:10.964179Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_current_delta": 5476836352, + "inactive_split_bytes_peak": 9402926080, + "inactive_split_bytes_peak_delta": 9242540032, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 96114573312, + "reserved_bytes_current_delta": 56870567936, + "reserved_bytes_peak": 96114573312, + "reserved_bytes_peak_delta": 56870567936, + "segment_count_current": 458, + "segment_count_current_delta": 387, + "segment_count_peak": 458, + "segment_count_peak_delta": 387, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T140619005110Z.pickle", + "timestamp_utc": "2026-08-01T14:06:19.120661Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T140619005110Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T140619005110Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..613a45b70fe95eec0522fbc66998a4a54cce3aaf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T140619005110Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:289b89e46429dfdb9f5e3442f6f40605c3a26ec84e5679b601518aa28985bcb7 +size 5941037 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T140620358500Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T140620358500Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ec5006f3fdf1f17e26f1a4015750014bd1e162a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T140620358500Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625008128, + "active_bytes_peak_delta": 23621444096, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625008128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625008128, + "device": 1, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_peak": 9402926080, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:20.358076Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625008128, + "allocated_bytes_peak_delta": 23621444096, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87003564032, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 87003564032, + "device": 1, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_peak": 9402926080, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 96114573312, + "reserved_bytes_peak": 96114573312, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.139266Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_current_delta": -121634816, + "inactive_split_bytes_peak": 9402926080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 16793993216, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 16793993216, + "segment_count_current": 535, + "segment_count_current_delta": 77, + "segment_count_peak": 535, + "segment_count_peak_delta": 77, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T140620358500Z.pickle", + "timestamp_utc": "2026-08-01T14:06:20.473650Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T140620358500Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T140620358500Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7976cde3199604bb7f3a20a8de291634f83e5192 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T140620358500Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:900b859150dd9ebdfc1ed970ea3c527d9df02258d43e7e61ca0bcbf14e4b75aa +size 5956341 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T140621629559Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T140621629559Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6a15e8a4060ea65563deb1558273eb227bf0a966 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T140621629559Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 178176, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 1, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_peak": 9402926080, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:21.629143Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 178176, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625008128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625008128, + "device": 1, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_peak": 9402926080, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:20.487972Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_current_delta": 1533018112, + "inactive_split_bytes_peak": 9402926080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T140621629559Z.pickle", + "timestamp_utc": "2026-08-01T14:06:21.734749Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T140621629559Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T140621629559Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..18ffae947486e2452126898c9e6563f8c728f2d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T140621629559Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b72f4846ef0fc0661bbbfbe2ff2a58c0cf9338a59f98ebaa4d88e2171acbfac +size 5957644 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T140622919471Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T140622919471Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3714b67e245300f7cc05b3905f74f3078ceb530 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T140622919471Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 1, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_peak": 9402926080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:22.919058Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 1, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_peak": 9402926080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:21.748373Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 9402926080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T140622919471Z.pickle", + "timestamp_utc": "2026-08-01T14:06:23.025896Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T140622919471Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T140622919471Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..73f467efab58d7a22cd0c244dffa3840a07ec089 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T140622919471Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb4a0995e9c703a6f970081ea4581e2adeb0c5ae6836b1687e0f53563ae5c7e6 +size 6025121 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T140624472431Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T140624472431Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e96e031893ed3703cbd81605425a1e5bffb80b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T140624472431Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 1, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_peak": 9402926080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:24.472021Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 1, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_peak": 9402926080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:23.039198Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_current_delta": -1115684864, + "inactive_split_bytes_peak": 9402926080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T140624472431Z.pickle", + "timestamp_utc": "2026-08-01T14:06:24.578922Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T140624472431Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T140624472431Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..37c1a431809a4e78c046ab52eb6273fc38c3f710 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T140624472431Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4394a5492dc85cf25806c2eb5f8caa4b3166d00e396f228f1237d5d49f2ca036 +size 5956390 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T140625766261Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T140625766261Z.json new file mode 100644 index 0000000000000000000000000000000000000000..08aa41cb5d5eb475d4731fec06905f98d28dc5b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T140625766261Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 1, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_peak": 9402926080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:25.765858Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 1, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_peak": 9402926080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:24.592429Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_current_delta": 819986432, + "inactive_split_bytes_peak": 9402926080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T140625766261Z.pickle", + "timestamp_utc": "2026-08-01T14:06:25.868309Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T140625766261Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T140625766261Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d34320c749dee1179f2e1eba3f8aff683e7caf1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T140625766261Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2306e55c548278590a7bed232830239ced11c2a88bd3eb622e076d93bc59c77 +size 5956258 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T140627053229Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T140627053229Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eec4f9165dd720940ff9c45b290eddfc856b08b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T140627053229Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 1, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_peak": 9402926080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:27.052819Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 1, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_peak": 9402926080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:25.881456Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_current_delta": -761266176, + "inactive_split_bytes_peak": 9402926080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T140627053229Z.pickle", + "timestamp_utc": "2026-08-01T14:06:27.155116Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T140627053229Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T140627053229Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9063ceea8aaafdb0646e748444d4496b5b37232 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T140627053229Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed21a9ebe53a19725cebcfbed4ca739f806608d7ede5967c897b86f693c07daa +size 6066813 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T140628495896Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T140628495896Z.json new file mode 100644 index 0000000000000000000000000000000000000000..45af82ecfad4abcf0e3984a8c6eef19b461ee0d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T140628495896Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627482624, + "active_bytes_peak_delta": 2296320, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 1, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_peak": 9402926080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:28.495504Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627482624, + "allocated_bytes_peak_delta": 2296320, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 1, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_peak": 9402926080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:27.168324Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_current_delta": 352321536, + "inactive_split_bytes_peak": 9402926080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T140628495896Z.pickle", + "timestamp_utc": "2026-08-01T14:06:28.599548Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T140628495896Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T140628495896Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..098061361a0920df81f7fc9c7aab347e6b6a55ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T140628495896Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76f2bf91cf199ec061d97720006b7b5885804d8131abdc1bfadb26a898c8c592 +size 6075553 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T140629778732Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T140629778732Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6d4b5df36b1301da7a1d701b63763215f86e5e9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T140629778732Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627482624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 1, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_peak": 9402926080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:29.778317Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627482624, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 1, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_peak": 9402926080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:28.612821Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_current_delta": 1704984576, + "inactive_split_bytes_peak": 9402926080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T140629778732Z.pickle", + "timestamp_utc": "2026-08-01T14:06:29.883436Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T140629778732Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T140629778732Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..83145c8f2ac3952d5b76985bd48940d7794704e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T140629778732Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a00f0bac29ed49bb1bc9aba563c5441faca5dec0a9dc5a2f3c1015bdea342e20 +size 5956045 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T140722684086Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T140722684086Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bcc5c4843c9141d96e643f08ca44115772f9ee22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T140722684086Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 110628334592, + "active_bytes_peak_delta": 851968, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 110628334592, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 110628334592, + "device": 1, + "inactive_split_bytes_current": 6779866112, + "inactive_split_bytes_peak": 9402926080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:07:22.683624Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110628334592, + "allocated_bytes_peak_delta": 851968, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 1, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_peak": 9402926080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:29.896935Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6779866112, + "inactive_split_bytes_current_delta": -1063256064, + "inactive_split_bytes_peak": 9402926080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T140722684086Z.pickle", + "timestamp_utc": "2026-08-01T14:07:22.884826Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T140722684086Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T140722684086Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9a677ec2a2c64629fcc2256bd472d3d7ab57a874 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T140722684086Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19eb270f33959149b1d6144d5d698162c1d20b9fd7e18fdaedc642d6e5e10b46 +size 9710511 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T140619005124Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T140619005124Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7254b5fd63172a2979f6772fbde94a6cbc76c77f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T140619005124Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 87003564032, + "active_bytes_peak_delta": 51537227776, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87003564032, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 87003564032, + "device": 2, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_peak": 9405023232, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 96114573312, + "reserved_bytes_peak": 96114573312, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.004540Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 87003564032, + "allocated_bytes_peak_delta": 51537227776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:05:10.964236Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_current_delta": 5476836352, + "inactive_split_bytes_peak": 9405023232, + "inactive_split_bytes_peak_delta": 9244637184, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 96114573312, + "reserved_bytes_current_delta": 56870567936, + "reserved_bytes_peak": 96114573312, + "reserved_bytes_peak_delta": 56870567936, + "segment_count_current": 458, + "segment_count_current_delta": 387, + "segment_count_peak": 458, + "segment_count_peak_delta": 387, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T140619005124Z.pickle", + "timestamp_utc": "2026-08-01T14:06:19.122814Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T140619005124Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T140619005124Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e894a1133b9827c6944482837d2fd6fded608008 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T140619005124Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5ee3b8702d095abb2b8e0394a192f4a4bd02959de330bfd87f4e92efc909bdb +size 5941039 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T140620358515Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T140620358515Z.json new file mode 100644 index 0000000000000000000000000000000000000000..17ab0095ca3d1766bb20ed9a174d0c59ecf657c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T140620358515Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625008128, + "active_bytes_peak_delta": 23621444096, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625008128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625008128, + "device": 2, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_peak": 9405023232, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:20.358081Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625008128, + "allocated_bytes_peak_delta": 23621444096, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87003564032, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 87003564032, + "device": 2, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_peak": 9405023232, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 96114573312, + "reserved_bytes_peak": 96114573312, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.143522Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_current_delta": -121634816, + "inactive_split_bytes_peak": 9405023232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 16793993216, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 16793993216, + "segment_count_current": 535, + "segment_count_current_delta": 77, + "segment_count_peak": 535, + "segment_count_peak_delta": 77, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T140620358515Z.pickle", + "timestamp_utc": "2026-08-01T14:06:20.478206Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T140620358515Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T140620358515Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a0452f6e89adfe55b654de3ced18aabc2bb1c452 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T140620358515Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfcc591a046a578a75e487f03333d184982c5a4628075f4986837dbc482440f5 +size 5956343 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T140621629540Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T140621629540Z.json new file mode 100644 index 0000000000000000000000000000000000000000..100eff19038fc544df34f095ce752c8432e81c4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T140621629540Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 178176, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 2, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_peak": 9405023232, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:21.629150Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 178176, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625008128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625008128, + "device": 2, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_peak": 9405023232, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:20.494119Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_current_delta": 1533018112, + "inactive_split_bytes_peak": 9405023232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T140621629540Z.pickle", + "timestamp_utc": "2026-08-01T14:06:21.743588Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T140621629540Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T140621629540Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6127eac3ca1beb5aa86378676194a5155a86a424 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T140621629540Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0e930ca72f95e6b2fe14266b75a3f3492bd66b63fbfd216fa7bd9289403128a +size 5957642 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T140622919418Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T140622919418Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76761793b69971e2541e8165ba788196d6ce0f96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T140622919418Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 2, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_peak": 9405023232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:22.919017Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 2, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_peak": 9405023232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:21.759260Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 9405023232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T140622919418Z.pickle", + "timestamp_utc": "2026-08-01T14:06:23.027445Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T140622919418Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T140622919418Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..90c57c59d89da6937f3c5ad3fcf95563a3a1c1e1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T140622919418Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6f16f0a5e035f00d8ab227fa8936dbbdbfc9d69b347e69ef1c96307fe9fda20 +size 6025116 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T140624472425Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T140624472425Z.json new file mode 100644 index 0000000000000000000000000000000000000000..165717cfdf3cb8862b1230e72088b1adf4feb586 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T140624472425Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 2, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_peak": 9405023232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:24.472013Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 2, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_peak": 9405023232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:23.042595Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_current_delta": -1115684864, + "inactive_split_bytes_peak": 9405023232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T140624472425Z.pickle", + "timestamp_utc": "2026-08-01T14:06:24.584388Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T140624472425Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T140624472425Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..52502430e75dce00b7ffeb5d5aea780bd2dd1e5e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T140624472425Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34f4e70eae855dad377a63bf2551a406b15a1e4a9c8b4b1ba0d1f602d7a58c84 +size 5956389 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T140625766265Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T140625766265Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d46fb54f16d518daf7558d85c9f3ec983d18099 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T140625766265Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 2, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_peak": 9405023232, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:25.765868Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 2, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_peak": 9405023232, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:24.602022Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_current_delta": 819986432, + "inactive_split_bytes_peak": 9405023232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T140625766265Z.pickle", + "timestamp_utc": "2026-08-01T14:06:25.871878Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T140625766265Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T140625766265Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f8df2c641b3f25f021ba7235cf1bc84dc6aac154 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T140625766265Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a475f72d9907d5b6ca1a6bd64744de346a585937ece79d80ed643d8779075300 +size 5956252 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T140627053228Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T140627053228Z.json new file mode 100644 index 0000000000000000000000000000000000000000..39ac35d1bb8dc28e739bc20455f64b25f8c66300 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T140627053228Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 2, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_peak": 9405023232, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:27.052819Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 2, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_peak": 9405023232, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:25.886805Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_current_delta": -761266176, + "inactive_split_bytes_peak": 9405023232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T140627053228Z.pickle", + "timestamp_utc": "2026-08-01T14:06:27.159020Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T140627053228Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T140627053228Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0b1fe71960491a446cd73caf3e5400567a96c4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T140627053228Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf9173629584ac4a0d39ecc045bd905a5fa950011dd4bea4f3fa989ce51148d0 +size 6066804 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T140628495919Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T140628495919Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4fe360a075a85e1dbcca34f6daad50359298d754 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T140628495919Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627482624, + "active_bytes_peak_delta": 2296320, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 2, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_peak": 9405023232, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:28.495506Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627482624, + "allocated_bytes_peak_delta": 2296320, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 2, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_peak": 9405023232, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:27.173992Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_current_delta": 352321536, + "inactive_split_bytes_peak": 9405023232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T140628495919Z.pickle", + "timestamp_utc": "2026-08-01T14:06:28.600780Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T140628495919Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T140628495919Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dd14465df24f8dc1c27978df5e40ad60a3ba6276 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T140628495919Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6872496482e9f31a4a8c1c6674caf891d01894a7dd6ac4a6228218df32801943 +size 6075545 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T140629778779Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T140629778779Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4ac5c4623c1c888e2e5e0e51cd3be9c52229920 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T140629778779Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627482624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 2, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_peak": 9405023232, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:29.778357Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627482624, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 2, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_peak": 9405023232, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:28.615656Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_current_delta": 1704984576, + "inactive_split_bytes_peak": 9405023232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T140629778779Z.pickle", + "timestamp_utc": "2026-08-01T14:06:29.885513Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T140629778779Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T140629778779Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9846da4e8d99962dfbb818c6d11a68ff2fce8b9a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T140629778779Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffeeec5fc83ef478b0e3fe0d9ac8d9e7e4ff4bde974b07999fdef250850dcbda +size 5956044 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T140722684084Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T140722684084Z.json new file mode 100644 index 0000000000000000000000000000000000000000..65d82f3ef053694bb435e5c22d2acf9add0a1e1d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T140722684084Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 110628334592, + "active_bytes_peak_delta": 851968, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 110628334592, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 110628334592, + "device": 2, + "inactive_split_bytes_current": 6779866112, + "inactive_split_bytes_peak": 9405023232, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:07:22.683628Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110628334592, + "allocated_bytes_peak_delta": 851968, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 2, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_peak": 9405023232, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:29.900587Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6779866112, + "inactive_split_bytes_current_delta": -1063256064, + "inactive_split_bytes_peak": 9405023232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T140722684084Z.pickle", + "timestamp_utc": "2026-08-01T14:07:22.891170Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T140722684084Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T140722684084Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7a9fd26bc65ea5fdd07fa36f8314164b1570c6d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T140722684084Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8981d193d8f42ca1cd668d83674768d8cc15b0f4c2d743a4476ebf5af3e4ed07 +size 9710511 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T140619005108Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T140619005108Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1654083d72bf219dc44668130401bd9eca0c7d02 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T140619005108Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 87003564032, + "active_bytes_peak_delta": 51537227776, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87003564032, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 87003564032, + "device": 3, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_peak": 9409217536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 96114573312, + "reserved_bytes_peak": 96114573312, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.004532Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 87003564032, + "allocated_bytes_peak_delta": 51537227776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:05:10.964193Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_current_delta": 5476836352, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 9248831488, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 96114573312, + "reserved_bytes_current_delta": 56870567936, + "reserved_bytes_peak": 96114573312, + "reserved_bytes_peak_delta": 56870567936, + "segment_count_current": 458, + "segment_count_current_delta": 387, + "segment_count_peak": 458, + "segment_count_peak_delta": 387, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T140619005108Z.pickle", + "timestamp_utc": "2026-08-01T14:06:19.121349Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T140619005108Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T140619005108Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1fae507917dcab46f27eaac6d19e3fe10d30424a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T140619005108Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68b7b348abc2fac8a9837e4335d15bb0a61c7ce0c564fc01e2d66b005829faee +size 5941030 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T140620358515Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T140620358515Z.json new file mode 100644 index 0000000000000000000000000000000000000000..93b16a50e720791ce7e02721333f2af20cc90429 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T140620358515Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625008128, + "active_bytes_peak_delta": 23621444096, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625008128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625008128, + "device": 3, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_peak": 9409217536, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:20.358081Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625008128, + "allocated_bytes_peak_delta": 23621444096, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87003564032, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 87003564032, + "device": 3, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_peak": 9409217536, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 96114573312, + "reserved_bytes_peak": 96114573312, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.139712Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_current_delta": -121634816, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 16793993216, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 16793993216, + "segment_count_current": 535, + "segment_count_current_delta": 77, + "segment_count_peak": 535, + "segment_count_peak_delta": 77, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T140620358515Z.pickle", + "timestamp_utc": "2026-08-01T14:06:20.474940Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T140620358515Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T140620358515Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8668ccd4a51cd1ee5953608ccf5810c960732edb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T140620358515Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f60bf48a4bedbcc8f3753e31a586d15a5badfb4442bc36239610ed1d8382d40 +size 5956341 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T140621629537Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T140621629537Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a54d207263e37dde27c1ac501e0fafea50b8e49 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T140621629537Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 178176, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 3, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_peak": 9409217536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:21.629142Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 178176, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625008128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625008128, + "device": 3, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_peak": 9409217536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:20.489260Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_current_delta": 1533018112, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T140621629537Z.pickle", + "timestamp_utc": "2026-08-01T14:06:21.737291Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T140621629537Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T140621629537Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..089bdfc7da1b4f835ce6b4f97755cae68c8f2916 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T140621629537Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5918cb68a7fb1058f8fa8481e681fa40e84641ab99f51688c33e0e92f77d998e +size 5957642 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T140622919444Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T140622919444Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bd5d77b31a002fe29199b89ddc454aa2b9bccd74 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T140622919444Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 3, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_peak": 9409217536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:22.919042Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 3, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_peak": 9409217536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:21.750921Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T140622919444Z.pickle", + "timestamp_utc": "2026-08-01T14:06:23.032166Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T140622919444Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T140622919444Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c0f9ec0fc58795c46d87f44211b56e3e44fe7da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T140622919444Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:281e54502e42fde1951e4bf1c159861ff76cf2d5543833eae2106bec574166ce +size 6025123 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T140624472447Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T140624472447Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e6925d64a81ee727def2d10557412cca02259c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T140624472447Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 3, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_peak": 9409217536, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:24.472032Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 3, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_peak": 9409217536, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:23.047894Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_current_delta": -1115684864, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T140624472447Z.pickle", + "timestamp_utc": "2026-08-01T14:06:24.580229Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T140624472447Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T140624472447Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..52ea91805e388f2f7ba1b8dbf701f7a5d330d4b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T140624472447Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2a5ee064e55e4b886701c90390bd464a514690649e56584dc525ee762141708 +size 5956390 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T140625766301Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T140625766301Z.json new file mode 100644 index 0000000000000000000000000000000000000000..deb855185e78e2856255ea2f8c043746e6027129 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T140625766301Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 3, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_peak": 9409217536, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:25.765890Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 3, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_peak": 9409217536, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:24.596263Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_current_delta": 819986432, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T140625766301Z.pickle", + "timestamp_utc": "2026-08-01T14:06:25.874014Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T140625766301Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T140625766301Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b4b31543700a35d9156921ae93c0e2cb3203591e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T140625766301Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c30690b5db05b82a2781e735db55b36e1e2628f6f0ecc157fb81b793e9c9d7e6 +size 5956253 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T140627053229Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T140627053229Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fdcc3962083e15049967d6766c646f5164a8c05a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T140627053229Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 3, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_peak": 9409217536, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:27.052819Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 3, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_peak": 9409217536, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:25.887364Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_current_delta": -761266176, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T140627053229Z.pickle", + "timestamp_utc": "2026-08-01T14:06:27.156510Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T140627053229Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T140627053229Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1bac6d6a77f4080ae5f27959e9e3f3fb9b94765d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T140627053229Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a4693ffdd06eb0eea61d8b87fa053aec7cc78e143cec1603f7534f0743e6bee +size 6066810 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T140628495920Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T140628495920Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f24732fc8303c1a2f8dd3e4e9d5bfbf0af5285f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T140628495920Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627482624, + "active_bytes_peak_delta": 2296320, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 3, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_peak": 9409217536, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:28.495513Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627482624, + "allocated_bytes_peak_delta": 2296320, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 3, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_peak": 9409217536, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:27.169644Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_current_delta": 352321536, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T140628495920Z.pickle", + "timestamp_utc": "2026-08-01T14:06:28.599783Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T140628495920Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T140628495920Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df32d155d7d3776eaf6824522a082041113d453f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T140628495920Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3434290b1ba93502c0a798dc5a00930eba4546267876c675198187d8e29a524 +size 6075550 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T140629778726Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T140629778726Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d0a0aa42bc46e80a1f6098312f43c981d59eaf86 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T140629778726Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627482624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 3, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_peak": 9409217536, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:29.778325Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627482624, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 3, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_peak": 9409217536, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:28.612934Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_current_delta": 1704984576, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T140629778726Z.pickle", + "timestamp_utc": "2026-08-01T14:06:29.884619Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T140629778726Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T140629778726Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3c6ea5d64217b17654113c87234154caed585bca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T140629778726Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e5e3ace1777447cf73c7952d2d000da88304398bb74d74f49c276064385bae4 +size 5956042 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T140722684096Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T140722684096Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6479869ddc8e90513396c7b6620e186c59396c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T140722684096Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 110628334592, + "active_bytes_peak_delta": 851968, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 110628334592, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 110628334592, + "device": 3, + "inactive_split_bytes_current": 6779866112, + "inactive_split_bytes_peak": 9409217536, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:07:22.683638Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110628334592, + "allocated_bytes_peak_delta": 851968, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 3, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_peak": 9409217536, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:29.898137Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6779866112, + "inactive_split_bytes_current_delta": -1063256064, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T140722684096Z.pickle", + "timestamp_utc": "2026-08-01T14:07:22.894554Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T140722684096Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T140722684096Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4596f3593bce2d33f033392ed93e72e693997399 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T140722684096Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fdc4ed6a3fb530ec9fb8e69b1ac86756ee0adeb8c2d052cc632af7844282be8 +size 9710511 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T140619005118Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T140619005118Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d117fdfd954616e027c000795b21dc0f95357577 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T140619005118Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441057280, + "active_bytes_current_delta": 37209159680, + "active_bytes_peak": 86998860800, + "active_bytes_peak_delta": 51532082176, + "after": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 86998860800, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 86998860800, + "device": 4, + "inactive_split_bytes_current": 4245499904, + "inactive_split_bytes_peak": 8120030208, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 95642714112, + "reserved_bytes_peak": 95642714112, + "segment_count_current": 453, + "segment_count_peak": 453, + "timestamp_utc": "2026-08-01T14:06:19.004535Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441057280, + "allocated_bytes_current_delta": 37209159680, + "allocated_bytes_peak": 86998860800, + "allocated_bytes_peak_delta": 51532082176, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:05:10.967979Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4245499904, + "inactive_split_bytes_current_delta": 4113123328, + "inactive_split_bytes_peak": 8120030208, + "inactive_split_bytes_peak_delta": 7966620672, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 95642714112, + "reserved_bytes_current_delta": 56398708736, + "reserved_bytes_peak": 95642714112, + "reserved_bytes_peak_delta": 56398708736, + "segment_count_current": 453, + "segment_count_current_delta": 382, + "segment_count_peak": 453, + "segment_count_peak_delta": 382, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T140619005118Z.pickle", + "timestamp_utc": "2026-08-01T14:06:19.124009Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T140619005118Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T140619005118Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..62582cf7d7264fa248f431cd08729e24b6d04ded --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T140619005118Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8c18e700aef7090280ff83f93b8167c865cc7863204b7d6cb221f9cb8b6dc28 +size 5948928 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T140620358487Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T140620358487Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7b0a42a079b0168caf11140f1e8fb3977cd859f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T140620358487Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441057280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110640386560, + "active_bytes_peak_delta": 23641525760, + "after": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110640386560, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110640386560, + "device": 4, + "inactive_split_bytes_current": 5302464512, + "inactive_split_bytes_peak": 8120030208, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:20.358066Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441057280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110640386560, + "allocated_bytes_peak_delta": 23641525760, + "before": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 86998860800, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 86998860800, + "device": 4, + "inactive_split_bytes_current": 4245499904, + "inactive_split_bytes_peak": 8120030208, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 95642714112, + "reserved_bytes_peak": 95642714112, + "segment_count_current": 453, + "segment_count_peak": 453, + "timestamp_utc": "2026-08-01T14:06:19.145186Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5302464512, + "inactive_split_bytes_current_delta": 1056964608, + "inactive_split_bytes_peak": 8120030208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112849846272, + "reserved_bytes_current_delta": 17207132160, + "reserved_bytes_peak": 112849846272, + "reserved_bytes_peak_delta": 17207132160, + "segment_count_current": 536, + "segment_count_current_delta": 83, + "segment_count_peak": 536, + "segment_count_peak_delta": 83, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T140620358487Z.pickle", + "timestamp_utc": "2026-08-01T14:06:20.479382Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T140620358487Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T140620358487Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..786065ddf5ab394e5dff53d4b6a3fc2df69b492d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T140620358487Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdd31963e3374851bc20017d921b12ada664c69df85c92e0472dd666a66470c5 +size 6631505 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T140621629539Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T140621629539Z.json new file mode 100644 index 0000000000000000000000000000000000000000..90bcb142b7c54996e2ca11ad89c9a91af7d7361d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T140621629539Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441057280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110640386560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110640386560, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110640386560, + "device": 4, + "inactive_split_bytes_current": 5069680640, + "inactive_split_bytes_peak": 8120030208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:21.629138Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441057280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110640386560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110640386560, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110640386560, + "device": 4, + "inactive_split_bytes_current": 5302464512, + "inactive_split_bytes_peak": 8120030208, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:20.496644Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5069680640, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 8120030208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112849846272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112849846272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T140621629539Z.pickle", + "timestamp_utc": "2026-08-01T14:06:21.738576Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T140621629539Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T140621629539Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13fe6293cfb0a410b3ae160d16611613cdb4e8db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T140621629539Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27f1cf86a179b62d4d69f5c1241b696a93c8ab76037995aba9ad1add70f7e566 +size 6628404 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T140622919438Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T140622919438Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad7f8930ea24f4c0cd71a3d2fc3f7f7b0f53e27f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T140622919438Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441057280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110640386560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110640386560, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110640386560, + "device": 4, + "inactive_split_bytes_current": 6124548096, + "inactive_split_bytes_peak": 8120030208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:22.919016Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441057280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110640386560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110640386560, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110640386560, + "device": 4, + "inactive_split_bytes_current": 5069680640, + "inactive_split_bytes_peak": 8120030208, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:21.754713Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6124548096, + "inactive_split_bytes_current_delta": 1054867456, + "inactive_split_bytes_peak": 8120030208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112849846272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112849846272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T140622919438Z.pickle", + "timestamp_utc": "2026-08-01T14:06:23.029742Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T140622919438Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T140622919438Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..851c534032b93f95e3dd99a4b2485e716a5d338a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T140622919438Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78e98a9285d2353ed6839949896f237bf720b23bf96aea959d9ae828d0eb0b5c +size 6645437 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T140624472407Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T140624472407Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c2a715e13d8ecaea34759b0603c8f10d67390abf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T140624472407Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441057280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110640386560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110640386560, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110640386560, + "device": 4, + "inactive_split_bytes_current": 5243744256, + "inactive_split_bytes_peak": 8120030208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:24.471984Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441057280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110640386560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110640386560, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110640386560, + "device": 4, + "inactive_split_bytes_current": 6124548096, + "inactive_split_bytes_peak": 8120030208, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:23.056990Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5243744256, + "inactive_split_bytes_current_delta": -880803840, + "inactive_split_bytes_peak": 8120030208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112849846272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112849846272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T140624472407Z.pickle", + "timestamp_utc": "2026-08-01T14:06:24.586532Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T140624472407Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T140624472407Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a61131cc682af61b69cce797e5a4322845e3b17c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T140624472407Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3029501ddf2314c8f8b60967e9a654b097ebd00e640a1e9e047fd7d238abd386 +size 6627104 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T140625766262Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T140625766262Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0c0ce1bc636b0b08918bde4ff065db4b8e399b20 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T140625766262Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441057280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110640386560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110640386560, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110640386560, + "device": 4, + "inactive_split_bytes_current": 5126303744, + "inactive_split_bytes_peak": 8120030208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:25.765846Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441057280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110640386560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110640386560, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110640386560, + "device": 4, + "inactive_split_bytes_current": 5243744256, + "inactive_split_bytes_peak": 8120030208, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:24.602570Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5126303744, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 8120030208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112849846272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112849846272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T140625766262Z.pickle", + "timestamp_utc": "2026-08-01T14:06:25.878876Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T140625766262Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T140625766262Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..67c7d1697d484cb9cf42ccdf42f0cc7fbd0b2564 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T140625766262Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10446fa4da77ea5c235ebafeb7a5617b4f6ca1dc9da2ed0ef5b7fca96f1a0b90 +size 6645139 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T140627053262Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T140627053262Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d0082e081968e8bbb2b78b1072b75bef44c0ab15 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T140627053262Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441057280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110640386560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110640386560, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110640386560, + "device": 4, + "inactive_split_bytes_current": 5539442688, + "inactive_split_bytes_peak": 8120030208, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:27.052824Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441057280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110640386560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110640386560, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110640386560, + "device": 4, + "inactive_split_bytes_current": 5126303744, + "inactive_split_bytes_peak": 8120030208, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:25.894905Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5539442688, + "inactive_split_bytes_current_delta": 413138944, + "inactive_split_bytes_peak": 8120030208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112849846272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112849846272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T140627053262Z.pickle", + "timestamp_utc": "2026-08-01T14:06:27.161064Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T140627053262Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T140627053262Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14a5599e31d0911c18af97add69b81fb92919245 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T140627053262Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47f302dcdb7084ef483e0dd21a151940dd6813c5e34ce48b512b97badf71f13f +size 6644998 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T140628495916Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T140628495916Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e5d6a1167941e1933ce103b7ac069c19dfb5a274 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T140628495916Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441057280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110641958912, + "active_bytes_peak_delta": 1572352, + "after": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110641958912, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110641958912, + "device": 4, + "inactive_split_bytes_current": 5243744256, + "inactive_split_bytes_peak": 8120030208, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:28.495483Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441057280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110641958912, + "allocated_bytes_peak_delta": 1572352, + "before": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110640386560, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110640386560, + "device": 4, + "inactive_split_bytes_current": 5539442688, + "inactive_split_bytes_peak": 8120030208, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:27.177141Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5243744256, + "inactive_split_bytes_current_delta": -295698432, + "inactive_split_bytes_peak": 8120030208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112849846272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112849846272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T140628495916Z.pickle", + "timestamp_utc": "2026-08-01T14:06:28.603849Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T140628495916Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T140628495916Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9492efd9b68062f8891a08c23e74f4f591f873e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T140628495916Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ea53a13bde98c6ea5db7cd29bbbb7eec0f0c2fff62db19385a3c78369383f66 +size 6626290 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T140629778716Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T140629778716Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf26b3ed042613e7447253aad0d15ecf32a97f46 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T140629778716Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441057280, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110641958912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110641958912, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110641958912, + "device": 4, + "inactive_split_bytes_current": 5243744256, + "inactive_split_bytes_peak": 8120030208, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:29.778299Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441057280, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110641958912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110641958912, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110641958912, + "device": 4, + "inactive_split_bytes_current": 5243744256, + "inactive_split_bytes_peak": 8120030208, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:28.619937Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5243744256, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8120030208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112849846272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112849846272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T140629778716Z.pickle", + "timestamp_utc": "2026-08-01T14:06:29.887404Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T140629778716Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T140629778716Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1181451bd1c3acd29c3d106fec36fc67626fbf04 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T140629778716Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5da6d3a5801f5cdb44072098d8726bd90cb345101bfa562d7d88281a0acd8641 +size 6626290 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T140722684091Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T140722684091Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0f5aa703b8b04b7e849290eb68ea08d5e6ce7515 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T140722684091Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166768128, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 110641958912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166768128, + "active_bytes_peak": 110641958912, + "allocated_bytes_current": 61166768128, + "allocated_bytes_peak": 110641958912, + "device": 4, + "inactive_split_bytes_current": 5002571776, + "inactive_split_bytes_peak": 8120030208, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:07:22.683624Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166768128, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110641958912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441057280, + "active_bytes_peak": 110641958912, + "allocated_bytes_current": 72441057280, + "allocated_bytes_peak": 110641958912, + "device": 4, + "inactive_split_bytes_current": 5243744256, + "inactive_split_bytes_peak": 8120030208, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 112849846272, + "reserved_bytes_peak": 112849846272, + "segment_count_current": 536, + "segment_count_peak": 536, + "timestamp_utc": "2026-08-01T14:06:29.903623Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5002571776, + "inactive_split_bytes_current_delta": -241172480, + "inactive_split_bytes_peak": 8120030208, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 112849846272, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112849846272, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 536, + "segment_count_current_delta": 0, + "segment_count_peak": 536, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T140722684091Z.pickle", + "timestamp_utc": "2026-08-01T14:07:22.887419Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T140722684091Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T140722684091Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2b2ea977782947686f14f644a3a9422dcaa81e4b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T140722684091Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ccb585a0e2f3e45f649ae9f46be60a647f2752b84f098a5b229b9be9df045e8 +size 10523194 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T140619005211Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T140619005211Z.json new file mode 100644 index 0000000000000000000000000000000000000000..74823ff843b901765190adfba467e8197b8f57e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T140619005211Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 87003564032, + "active_bytes_peak_delta": 51537227776, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87003564032, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 87003564032, + "device": 5, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_peak": 9407120384, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 96114573312, + "reserved_bytes_peak": 96114573312, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.004565Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 87003564032, + "allocated_bytes_peak_delta": 51537227776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:05:10.964773Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_current_delta": 5476836352, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 9246734336, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 96114573312, + "reserved_bytes_current_delta": 56870567936, + "reserved_bytes_peak": 96114573312, + "reserved_bytes_peak_delta": 56870567936, + "segment_count_current": 458, + "segment_count_current_delta": 387, + "segment_count_peak": 458, + "segment_count_peak_delta": 387, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T140619005211Z.pickle", + "timestamp_utc": "2026-08-01T14:06:19.119177Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T140619005211Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T140619005211Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4e78f323d3531c3e89ec607cf50e2aaa2adbb6a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T140619005211Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51cdf90d247f73eab86e013b6cb7087c6925b2c7c5092aeed86684c0bb07e575 +size 5941037 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T140620358505Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T140620358505Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9eca3787d67f5a71d394b86ff1018ac7f8faad0f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T140620358505Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625008128, + "active_bytes_peak_delta": 23621444096, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625008128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625008128, + "device": 5, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_peak": 9407120384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:20.358087Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625008128, + "allocated_bytes_peak_delta": 23621444096, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87003564032, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 87003564032, + "device": 5, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_peak": 9407120384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 96114573312, + "reserved_bytes_peak": 96114573312, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.134757Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_current_delta": -121634816, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 16793993216, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 16793993216, + "segment_count_current": 535, + "segment_count_current_delta": 77, + "segment_count_peak": 535, + "segment_count_peak_delta": 77, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T140620358505Z.pickle", + "timestamp_utc": "2026-08-01T14:06:20.471792Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T140620358505Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T140620358505Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0f131803018bff5756ef32989819bddf341df943 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T140620358505Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45ef8e7bcb5a767ed289b517ab19cef8bd061f8383f003ca097984eb7015625a +size 5956342 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T140621629559Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T140621629559Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd80ff074d288993a6a2529bc4fa62689f02c4f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T140621629559Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 178176, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 5, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_peak": 9407120384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:21.629158Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 178176, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625008128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625008128, + "device": 5, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_peak": 9407120384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:20.484259Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_current_delta": 1533018112, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T140621629559Z.pickle", + "timestamp_utc": "2026-08-01T14:06:21.727230Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T140621629559Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T140621629559Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..151caeb11b3db43642e7ab3af24184aee0f2a702 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T140621629559Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:338dac27550907bb2dfcb745bc276b953ec039dfcf4dde9386c085412010e46f +size 5957640 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T140622919446Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T140622919446Z.json new file mode 100644 index 0000000000000000000000000000000000000000..49c83c51587d9da5c41d26b3a4e73e2198aedc4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T140622919446Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 5, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_peak": 9407120384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:22.919043Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 5, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_peak": 9407120384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:21.738759Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T140622919446Z.pickle", + "timestamp_utc": "2026-08-01T14:06:23.018122Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T140622919446Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T140622919446Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e5a1ec610d2cafb988eb538612fbe2a3f9c6f77 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T140622919446Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97c3a0bc8dd120fa02625f5ab8e4f877ca38710793874558d9bb2633264b98e0 +size 6025130 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T140624472436Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T140624472436Z.json new file mode 100644 index 0000000000000000000000000000000000000000..867311e37c2ef24797e0f121f5caddcacb73aa66 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T140624472436Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 5, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_peak": 9407120384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:24.472027Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 5, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_peak": 9407120384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:23.031695Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_current_delta": -1115684864, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T140624472436Z.pickle", + "timestamp_utc": "2026-08-01T14:06:24.577631Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T140624472436Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T140624472436Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8579177b3eb92817b7567c2e37b22d0dbb2d7cee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T140624472436Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1daf73b6ece1a4c4dc2ee9eb2b59149f05658f9f16547c81bb43a1b03b03aac +size 5956389 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T140625766290Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T140625766290Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cfd0fdb2e6aad805a7249fb9204e2034883eec9a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T140625766290Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 5, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_peak": 9407120384, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:25.765891Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 5, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_peak": 9407120384, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:24.589822Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_current_delta": 819986432, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T140625766290Z.pickle", + "timestamp_utc": "2026-08-01T14:06:25.868117Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T140625766290Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T140625766290Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b511dbcf41de51d6eef78934fba00a38df443f98 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T140625766290Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:683d4f5ef328d527cf4f01f7eda402f515216aad68f3044f5aa431cefbba70ff +size 5956252 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T140627053232Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T140627053232Z.json new file mode 100644 index 0000000000000000000000000000000000000000..84bd0ef3b2c6d240d8e980bb3591f5b44e9f8ef2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T140627053232Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 5, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_peak": 9407120384, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:27.052828Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 5, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_peak": 9407120384, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:25.881963Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_current_delta": -761266176, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T140627053232Z.pickle", + "timestamp_utc": "2026-08-01T14:06:27.150024Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T140627053232Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T140627053232Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1aca1048ccf29c4b212b5be747229567174478ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T140627053232Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caef997f22e3ef07b6b5149b67ec88e2eec2d6136e8eb81da1c5d08af1b8a6c0 +size 6066816 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T140628495906Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T140628495906Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7fdbd3d9dba22142b43bacab565eadf4d21ac1cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T140628495906Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627482624, + "active_bytes_peak_delta": 2296320, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 5, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_peak": 9407120384, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:28.495510Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627482624, + "allocated_bytes_peak_delta": 2296320, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 5, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_peak": 9407120384, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:27.161519Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_current_delta": 352321536, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T140628495906Z.pickle", + "timestamp_utc": "2026-08-01T14:06:28.595113Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T140628495906Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T140628495906Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b23a3f449f22c10f61864993c56d99c485db228 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T140628495906Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:905a803aa264f9588881e1568714288f29039e87ef6c274870d83c2b2ba63ba3 +size 6075550 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T140629778740Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T140629778740Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d080c8694182aaa518f69a24fb601878683146b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T140629778740Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627482624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 5, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_peak": 9407120384, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:29.778329Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627482624, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 5, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_peak": 9407120384, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:28.606976Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_current_delta": 1704984576, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T140629778740Z.pickle", + "timestamp_utc": "2026-08-01T14:06:29.881198Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T140629778740Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T140629778740Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1882b12fe2dde69c910392c73efec3db5402ea64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T140629778740Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14f8659ce7808c259b929a81656102b723ebefbc1c0412d0184c4d5d6d151114 +size 5956048 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T140722684078Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T140722684078Z.json new file mode 100644 index 0000000000000000000000000000000000000000..331683832212e09f29b3b6029fb381c514e794f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T140722684078Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 110628334592, + "active_bytes_peak_delta": 851968, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 110628334592, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 110628334592, + "device": 5, + "inactive_split_bytes_current": 6779866112, + "inactive_split_bytes_peak": 9407120384, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:07:22.683623Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110628334592, + "allocated_bytes_peak_delta": 851968, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 5, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_peak": 9407120384, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:29.893281Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6779866112, + "inactive_split_bytes_current_delta": -1063256064, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T140722684078Z.pickle", + "timestamp_utc": "2026-08-01T14:07:22.894892Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T140722684078Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T140722684078Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3294e4220144cece699af7a8740b2cf77dc15d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T140722684078Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a6107c78867fdfd53a172ef3f6490d8e83836bc0e95c2a89adef843d16004bc +size 9710511 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T140619005191Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T140619005191Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34b5363e72090ade1a67b54d1bdd824ba72b9bfc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T140619005191Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 87003564032, + "active_bytes_peak_delta": 51537227776, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87003564032, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 87003564032, + "device": 6, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_peak": 9409217536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 96114573312, + "reserved_bytes_peak": 96114573312, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.004559Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 87003564032, + "allocated_bytes_peak_delta": 51537227776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:05:10.964146Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_current_delta": 5476836352, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 9248831488, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 96114573312, + "reserved_bytes_current_delta": 56870567936, + "reserved_bytes_peak": 96114573312, + "reserved_bytes_peak_delta": 56870567936, + "segment_count_current": 458, + "segment_count_current_delta": 387, + "segment_count_peak": 458, + "segment_count_peak_delta": 387, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T140619005191Z.pickle", + "timestamp_utc": "2026-08-01T14:06:19.121193Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T140619005191Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T140619005191Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6bcde7f6a99f05356aed8769600cad513ebab075 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T140619005191Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:234a9658a720850e44ca9021d20becfb29d24527dc232ba3ab3e00463b287c5b +size 5941037 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T140620358521Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T140620358521Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c37965c4b44fd8e6ae38b0cc0a3e322ef9b9caf9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T140620358521Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625008128, + "active_bytes_peak_delta": 23621444096, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625008128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625008128, + "device": 6, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_peak": 9409217536, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:20.358093Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625008128, + "allocated_bytes_peak_delta": 23621444096, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87003564032, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 87003564032, + "device": 6, + "inactive_split_bytes_current": 5609655296, + "inactive_split_bytes_peak": 9409217536, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 96114573312, + "reserved_bytes_peak": 96114573312, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.139819Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_current_delta": -121634816, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 16793993216, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 16793993216, + "segment_count_current": 535, + "segment_count_current_delta": 77, + "segment_count_peak": 535, + "segment_count_peak_delta": 77, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T140620358521Z.pickle", + "timestamp_utc": "2026-08-01T14:06:20.473812Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T140620358521Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T140620358521Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..04aa52f83f27af24c81c6f3b3f4e4384d3b7ec36 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T140620358521Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e89173be8cbe958bd1b77361a537393ac0d41d8d9c540fc0597016901e43af58 +size 5956346 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T140621629592Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T140621629592Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2958b67a85d9536decdeed20600ece6aa162772c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T140621629592Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 178176, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 6, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_peak": 9409217536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:21.629175Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 178176, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625008128, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625008128, + "device": 6, + "inactive_split_bytes_current": 5488020480, + "inactive_split_bytes_peak": 9409217536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:20.487885Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_current_delta": 1533018112, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T140621629592Z.pickle", + "timestamp_utc": "2026-08-01T14:06:21.738202Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T140621629592Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T140621629592Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3396b908f910fc09be33dd402037ba5b05aae3b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T140621629592Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f836812a79919962a4d21dda76f3fbf62053138190d3ce04fae292145ed84fb +size 5957642 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T140622919434Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T140622919434Z.json new file mode 100644 index 0000000000000000000000000000000000000000..30f8733b479af6723b560aeb51c2fd99363fbeef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T140622919434Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 6, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_peak": 9409217536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:22.919013Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 6, + "inactive_split_bytes_current": 7021038592, + "inactive_split_bytes_peak": 9409217536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:21.751991Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T140622919434Z.pickle", + "timestamp_utc": "2026-08-01T14:06:23.036021Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T140622919434Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T140622919434Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6daaca1f2b95a60bf1fb1d9edb6a7c91172fd95d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T140622919434Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff41d9cab2902487a280a3185c6a1d1e3debcaa55fbd89353cf29f930cd90504 +size 6025123 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T140624472459Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T140624472459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..47a797799fd5913c969e158ae21a24ce41ff9b5d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T140624472459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 6, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_peak": 9409217536, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:24.472044Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 6, + "inactive_split_bytes_current": 6842780672, + "inactive_split_bytes_peak": 9409217536, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:23.052135Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_current_delta": -1115684864, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T140624472459Z.pickle", + "timestamp_utc": "2026-08-01T14:06:24.579911Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T140624472459Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T140624472459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..78bc67fd15e56a309502fb832485f1b5a970be88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T140624472459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:947283ad632db57cee59bc47c73250f06567a54d43c22db763acd9ad91070003 +size 5956390 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T140625770588Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T140625770588Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d836c3fee429e13bd68cd795e805b4853e62a9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T140625770588Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 6, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_peak": 9409217536, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:25.765906Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 6, + "inactive_split_bytes_current": 5727095808, + "inactive_split_bytes_peak": 9409217536, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:24.593520Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_current_delta": 819986432, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T140625770588Z.pickle", + "timestamp_utc": "2026-08-01T14:06:25.875994Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T140625770588Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T140625770588Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4a965615f2a56c8fc15185bbcd1bcc301d1e9b4a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T140625770588Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f343a34dd3e091c973be1786bb90c663a0e28851d75f66f6abe4ad347e4e3b7 +size 5956253 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T140627053236Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T140627053236Z.json new file mode 100644 index 0000000000000000000000000000000000000000..64960d9536d61ec7d9eeb90a6544a981658133db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T140627053236Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 6, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_peak": 9409217536, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:27.052836Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 6, + "inactive_split_bytes_current": 6547082240, + "inactive_split_bytes_peak": 9409217536, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:25.890013Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_current_delta": -761266176, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T140627053236Z.pickle", + "timestamp_utc": "2026-08-01T14:06:27.158772Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T140627053236Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T140627053236Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ec3a3f4935882f8a545b7ce4b8c8a8f6cebb8a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T140627053236Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d3aac31c39b0103b3414a732745d2d4670682aa7caf0d84250aa4c385014360 +size 6066806 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T140628495912Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T140628495912Z.json new file mode 100644 index 0000000000000000000000000000000000000000..943fbd712f4277bc3de22475388da2a594cc6c00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T140628495912Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627482624, + "active_bytes_peak_delta": 2296320, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 6, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_peak": 9409217536, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:28.495509Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627482624, + "allocated_bytes_peak_delta": 2296320, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110625186304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625186304, + "device": 6, + "inactive_split_bytes_current": 5785816064, + "inactive_split_bytes_peak": 9409217536, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:27.172439Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_current_delta": 352321536, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T140628495912Z.pickle", + "timestamp_utc": "2026-08-01T14:06:28.600236Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T140628495912Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T140628495912Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..515ea4ed1d5f2099dbc85206cc7523b01bd1f1e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T140628495912Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f7e3bc22d6a1dc4719c2b2bc2b6cf886eede0f4324b6d097b5b67c13556a666 +size 6075550 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T140629778749Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T140629778749Z.json new file mode 100644 index 0000000000000000000000000000000000000000..972647f214cc1e6e75c1e2e3da32b15a648fc705 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T140629778749Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627482624, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 6, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_peak": 9409217536, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:29.778344Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627482624, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 6, + "inactive_split_bytes_current": 6138137600, + "inactive_split_bytes_peak": 9409217536, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:28.613723Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_current_delta": 1704984576, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T140629778749Z.pickle", + "timestamp_utc": "2026-08-01T14:06:29.884045Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T140629778749Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T140629778749Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eb065b00c74ed0e1e322f1d4b4b555e77a82f4ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T140629778749Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b6859c7159e65b0685399c3ab9adb135c29439b6880fdddb5788ab6d2c9371e +size 5956045 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T140722684077Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T140722684077Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d0773edb76d58d93bff198d7af97f84809f14eaf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T140722684077Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 110628334592, + "active_bytes_peak_delta": 851968, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 110628334592, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 110628334592, + "device": 6, + "inactive_split_bytes_current": 6779866112, + "inactive_split_bytes_peak": 9409217536, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:07:22.683622Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110628334592, + "allocated_bytes_peak_delta": 851968, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110627482624, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627482624, + "device": 6, + "inactive_split_bytes_current": 7843122176, + "inactive_split_bytes_peak": 9409217536, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:29.897702Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6779866112, + "inactive_split_bytes_current_delta": -1063256064, + "inactive_split_bytes_peak": 9409217536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T140722684077Z.pickle", + "timestamp_utc": "2026-08-01T14:07:22.892634Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T140722684077Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T140722684077Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e0e5f1e30c3aa3f30285bcbf54a0592bf70e668 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T140722684077Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14b5719b62f464d48f662c30c7d0c998ea40e7eff2075019055c8c1d79de6df8 +size 9710513 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T140619005093Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T140619005093Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a038c0daea38ba707b6082f2d2582218c35839fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T140619005093Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 37208595968, + "active_bytes_peak": 87003564032, + "active_bytes_peak_delta": 51537227776, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 87003564032, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 87003564032, + "device": 7, + "inactive_split_bytes_current": 5609654784, + "inactive_split_bytes_peak": 9407120384, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 96114573312, + "reserved_bytes_peak": 96114573312, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.004528Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 37208595968, + "allocated_bytes_peak": 87003564032, + "allocated_bytes_peak_delta": 51537227776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:05:10.966261Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5609654784, + "inactive_split_bytes_current_delta": 5476835840, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 9246734336, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 96114573312, + "reserved_bytes_current_delta": 56870567936, + "reserved_bytes_peak": 96114573312, + "reserved_bytes_peak_delta": 56870567936, + "segment_count_current": 458, + "segment_count_current_delta": 387, + "segment_count_peak": 458, + "segment_count_peak_delta": 387, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T140619005093Z.pickle", + "timestamp_utc": "2026-08-01T14:06:19.122211Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T140619005093Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T140619005093Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2a8620514a3648eb88b45baf085331c23c36ad4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T140619005093Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15802b7c4013919fcd788842b4db1410c00e7b6717c855d4b0c16d7c0c4faa4b +size 5942623 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T140620358535Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T140620358535Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6b1a77f4bc7b5e76902014c388228d7f5df3e7e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T140620358535Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625008640, + "active_bytes_peak_delta": 23621444608, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110625008640, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110625008640, + "device": 7, + "inactive_split_bytes_current": 5488019968, + "inactive_split_bytes_peak": 9407120384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:20.358116Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625008640, + "allocated_bytes_peak_delta": 23621444608, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 87003564032, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 87003564032, + "device": 7, + "inactive_split_bytes_current": 5609654784, + "inactive_split_bytes_peak": 9407120384, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 96114573312, + "reserved_bytes_peak": 96114573312, + "segment_count_current": 458, + "segment_count_peak": 458, + "timestamp_utc": "2026-08-01T14:06:19.140959Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5488019968, + "inactive_split_bytes_current_delta": -121634816, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 16793993216, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 16793993216, + "segment_count_current": 535, + "segment_count_current_delta": 77, + "segment_count_peak": 535, + "segment_count_peak_delta": 77, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T140620358535Z.pickle", + "timestamp_utc": "2026-08-01T14:06:20.476869Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T140620358535Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T140620358535Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a164cbec1bf213fd68c7b161ba70568b08027641 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T140620358535Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8342f1074ec374e17bec0eba8a395e435c4341821d3d87da17f7fec0990c53c3 +size 5958093 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T140621629571Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T140621629571Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f17fb67ee6c841122b37fd9cb31f67360ba9735 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T140621629571Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186816, + "active_bytes_peak_delta": 178176, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110625186816, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110625186816, + "device": 7, + "inactive_split_bytes_current": 7021038080, + "inactive_split_bytes_peak": 9407120384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:21.629172Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186816, + "allocated_bytes_peak_delta": 178176, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110625008640, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110625008640, + "device": 7, + "inactive_split_bytes_current": 5488019968, + "inactive_split_bytes_peak": 9407120384, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:20.492961Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7021038080, + "inactive_split_bytes_current_delta": 1533018112, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T140621629571Z.pickle", + "timestamp_utc": "2026-08-01T14:06:21.740076Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T140621629571Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T140621629571Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1d6d294c24fd79705568c4ad8c62dbde77958453 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T140621629571Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:580bcb4fb0fbc3a2971b85277a305edcaf453e312dffe94107c2a2c36309359b +size 5959393 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T140622919603Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T140622919603Z.json new file mode 100644 index 0000000000000000000000000000000000000000..33e161f568ff41bcbd21b66ae6a9a322b88a83f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T140622919603Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186816, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110625186816, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110625186816, + "device": 7, + "inactive_split_bytes_current": 6842780160, + "inactive_split_bytes_peak": 9407120384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:22.919171Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110625186816, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110625186816, + "device": 7, + "inactive_split_bytes_current": 7021038080, + "inactive_split_bytes_peak": 9407120384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:21.755681Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6842780160, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T140622919603Z.pickle", + "timestamp_utc": "2026-08-01T14:06:23.038457Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T140622919603Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T140622919603Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..59065e6b09223bf6187b7f566456c0e3a6a50e43 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T140622919603Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:737b1edbee3cf1b183c51c74a7631d753070f7448c3e3078ab2892624a430983 +size 6026838 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T140624472408Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T140624472408Z.json new file mode 100644 index 0000000000000000000000000000000000000000..814b992e1fbed26d6c8b7a02e8beed7a11aa040f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T140624472408Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186816, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110625186816, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110625186816, + "device": 7, + "inactive_split_bytes_current": 5727095296, + "inactive_split_bytes_peak": 9407120384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:24.471980Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110625186816, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110625186816, + "device": 7, + "inactive_split_bytes_current": 6842780160, + "inactive_split_bytes_peak": 9407120384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:23.056622Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5727095296, + "inactive_split_bytes_current_delta": -1115684864, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T140624472408Z.pickle", + "timestamp_utc": "2026-08-01T14:06:24.583022Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T140624472408Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T140624472408Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..025a6732783bdef6fe3237a3fbbdeab70cbed40e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T140624472408Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:343a041d244d3b9ccd26614f2e1faef2b52a8c95148e06b1e0a6641dd672e03c +size 5958132 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T140625766294Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T140625766294Z.json new file mode 100644 index 0000000000000000000000000000000000000000..104eaf4f04b63a4c34424881a1d93b688503e2e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T140625766294Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186816, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110625186816, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110625186816, + "device": 7, + "inactive_split_bytes_current": 6547081728, + "inactive_split_bytes_peak": 9407120384, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:25.765885Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110625186816, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110625186816, + "device": 7, + "inactive_split_bytes_current": 5727095296, + "inactive_split_bytes_peak": 9407120384, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:24.598690Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6547081728, + "inactive_split_bytes_current_delta": 819986432, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T140625766294Z.pickle", + "timestamp_utc": "2026-08-01T14:06:25.880022Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T140625766294Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T140625766294Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d978f4d72e45ca3d012fbd2eb5a31a4f8cf79584 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T140625766294Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74c316b3184bb0b5546f1d64c7a2e258cd53398980931ed2351ad9ffa79f6803 +size 5958000 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T140627053251Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T140627053251Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f8e276438a286f347b17537145bb7404a9eb59c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T140627053251Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110625186816, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110625186816, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110625186816, + "device": 7, + "inactive_split_bytes_current": 5785815552, + "inactive_split_bytes_peak": 9407120384, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:27.052851Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625186816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110625186816, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110625186816, + "device": 7, + "inactive_split_bytes_current": 6547081728, + "inactive_split_bytes_peak": 9407120384, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:25.896063Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5785815552, + "inactive_split_bytes_current_delta": -761266176, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T140627053251Z.pickle", + "timestamp_utc": "2026-08-01T14:06:27.163094Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T140627053251Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T140627053251Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5cc24b1c309c4c1209586da734a45236d15dd536 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T140627053251Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b47373567e20f3931877f2cc22bd71c21475b7b51af86e1f44efd8e5487157a +size 6068533 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T140628495935Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T140628495935Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e03b8675dffc90d4644927f4145fea5f147c671a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T140628495935Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627483136, + "active_bytes_peak_delta": 2296320, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110627483136, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110627483136, + "device": 7, + "inactive_split_bytes_current": 6138137088, + "inactive_split_bytes_peak": 9407120384, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:28.495536Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627483136, + "allocated_bytes_peak_delta": 2296320, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110625186816, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110625186816, + "device": 7, + "inactive_split_bytes_current": 5785815552, + "inactive_split_bytes_peak": 9407120384, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:27.178626Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6138137088, + "inactive_split_bytes_current_delta": 352321536, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T140628495935Z.pickle", + "timestamp_utc": "2026-08-01T14:06:28.605430Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T140628495935Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T140628495935Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..10b58ec5e2f7c92ca9642fba0d909382ccd825b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T140628495935Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ebcbdd4d48d51bba80829973a3e8d317d9f866b5873867d4e9835830ed44076 +size 6077298 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T140629778781Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T140629778781Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5224463b235cef785b0d70abc4a1bd9982a2160 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T140629778781Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110627483136, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110627483136, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110627483136, + "device": 7, + "inactive_split_bytes_current": 7843121664, + "inactive_split_bytes_peak": 9407120384, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:29.778360Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627483136, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110627483136, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110627483136, + "device": 7, + "inactive_split_bytes_current": 6138137088, + "inactive_split_bytes_peak": 9407120384, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:28.620802Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7843121664, + "inactive_split_bytes_current_delta": 1704984576, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T140629778781Z.pickle", + "timestamp_utc": "2026-08-01T14:06:29.888078Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T140629778781Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T140629778781Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fc952b3c6cbf85a8da132bb77487bb99ec2c15fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T140629778781Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef26c8f0f90c677f9e07021b65ec7e989e658e11ef703d4d15ff3cc1ebada2da +size 5957796 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T140722684084Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T140722684084Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e125e8f47994b89027eafcd2f03a8b499d29f313 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T140722684084Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165762048, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 110628335104, + "active_bytes_peak_delta": 851968, + "after": { + "active_bytes_current": 61165762048, + "active_bytes_peak": 110628335104, + "allocated_bytes_current": 61165762048, + "allocated_bytes_peak": 110628335104, + "device": 7, + "inactive_split_bytes_current": 6779865600, + "inactive_split_bytes_peak": 9407120384, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:07:22.683628Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165762048, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110628335104, + "allocated_bytes_peak_delta": 851968, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110627483136, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110627483136, + "device": 7, + "inactive_split_bytes_current": 7843121664, + "inactive_split_bytes_peak": 9407120384, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 112908566528, + "reserved_bytes_peak": 112908566528, + "segment_count_current": 535, + "segment_count_peak": 535, + "timestamp_utc": "2026-08-01T14:06:29.903503Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6779865600, + "inactive_split_bytes_current_delta": -1063256064, + "inactive_split_bytes_peak": 9407120384, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 112908566528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 112908566528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 535, + "segment_count_current_delta": 0, + "segment_count_peak": 535, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T140722684084Z.pickle", + "timestamp_utc": "2026-08-01T14:07:22.884448Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T140722684084Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T140722684084Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2d021bd9518a89dd52fee20aa17defaa426d9a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T140722684084Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4af3e5d945874fd1e81f96365341315c226e8779fc5de9b00d73a5a72e09e59 +size 9712234 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..284c717161b98ef230a465099ed6c2da0389e3fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 4096 +capacity: 1.00 +expert_dispatch_overlap: on +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6038 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 226 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T141008516821Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T141008516821Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5489ecfff0522816b03ae6e18c251a12aa8c6778 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T141008516821Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 87096445952, + "active_bytes_peak_delta": 51630109696, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 87096445952, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 87096445952, + "device": 0, + "inactive_split_bytes_current": 5707910144, + "inactive_split_bytes_peak": 8378553344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 96338968576, + "reserved_bytes_peak": 96338968576, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:10:08.516407Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 87096445952, + "allocated_bytes_peak_delta": 51630109696, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T14:09:01.298459Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5707910144, + "inactive_split_bytes_current_delta": 5572994048, + "inactive_split_bytes_peak": 8378553344, + "inactive_split_bytes_peak_delta": 8217921536, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 96338968576, + "reserved_bytes_current_delta": 57092866048, + "reserved_bytes_peak": 96338968576, + "reserved_bytes_peak_delta": 57092866048, + "segment_count_current": 499, + "segment_count_current_delta": 427, + "segment_count_peak": 499, + "segment_count_peak_delta": 427, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T141008516821Z.pickle", + "timestamp_utc": "2026-08-01T14:10:08.630879Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T141008516821Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T141008516821Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..95a9ff44d312a4aa615dfeca76b658a939fafe6e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T141008516821Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d1e2649fd0031a71a1181f3ab5551e603e905f48fcbce15946d689799059833 +size 5409983 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T141009707878Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T141009707878Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cad42ba559bb025fd9f5eb6ac703293cffdb1b4b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T141009707878Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110725163008, + "active_bytes_peak_delta": 23628717056, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110725163008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110725163008, + "device": 0, + "inactive_split_bytes_current": 5726784512, + "inactive_split_bytes_peak": 8378553344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:09.707471Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110725163008, + "allocated_bytes_peak_delta": 23628717056, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 87096445952, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 87096445952, + "device": 0, + "inactive_split_bytes_current": 5707910144, + "inactive_split_bytes_peak": 8378553344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 96338968576, + "reserved_bytes_peak": 96338968576, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:10:08.647785Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5726784512, + "inactive_split_bytes_current_delta": 18874368, + "inactive_split_bytes_peak": 8378553344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_current_delta": 17205035008, + "reserved_bytes_peak": 113544003584, + "reserved_bytes_peak_delta": 17205035008, + "segment_count_current": 570, + "segment_count_current_delta": 71, + "segment_count_peak": 570, + "segment_count_peak_delta": 71, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T141009707878Z.pickle", + "timestamp_utc": "2026-08-01T14:10:09.824869Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T141009707878Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T141009707878Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..78a83a30952fec373f45e02ca6693f248c334d23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T141009707878Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ff518b92b0b8403f56504ec71647b4cf8c192e5d5d46ac93b5aebbebe8dae0d +size 5632350 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T141010846005Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T141010846005Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dda9c0e5ec6ce3c59e6ba6bc848b935e56b0d9c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T141010846005Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110733676544, + "active_bytes_peak_delta": 8513536, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5489806336, + "inactive_split_bytes_peak": 8378553344, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:10.845585Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110733676544, + "allocated_bytes_peak_delta": 8513536, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110725163008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110725163008, + "device": 0, + "inactive_split_bytes_current": 5726784512, + "inactive_split_bytes_peak": 8378553344, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:09.840563Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5489806336, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 8378553344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113544003584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 570, + "segment_count_current_delta": 0, + "segment_count_peak": 570, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T141010846005Z.pickle", + "timestamp_utc": "2026-08-01T14:10:10.950313Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T141010846005Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T141010846005Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a549b60dea0c59be744884f100cda6968cc3cbd6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T141010846005Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1952d44994de0e7d930ec0657dec59b4da9133f775f1f4406ccb5f06ab2a867 +size 5631389 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T141011997564Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T141011997564Z.json new file mode 100644 index 0000000000000000000000000000000000000000..981a00780a116d2af769c907424e020899fc8794 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T141011997564Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110733676544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5485612032, + "inactive_split_bytes_peak": 8378553344, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:11.997140Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110733676544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5489806336, + "inactive_split_bytes_peak": 8378553344, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:10.963145Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5485612032, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 8378553344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113544003584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 570, + "segment_count_current_delta": 0, + "segment_count_peak": 570, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T141011997564Z.pickle", + "timestamp_utc": "2026-08-01T14:10:12.100408Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T141011997564Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T141011997564Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..162f686c4fc51c03cf416201dc34dd0512fb8a6d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T141011997564Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d81e938f9de2504abe153cc397c60bba5d2517351e6f6f558b1f731beb8a9e5b +size 5632661 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T141013452080Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T141013452080Z.json new file mode 100644 index 0000000000000000000000000000000000000000..644729798e9e529a5f7cd0a8bb1ee2f626ef0f61 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T141013452080Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110733676544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5479320576, + "inactive_split_bytes_peak": 8378553344, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:13.451670Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110733676544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5485612032, + "inactive_split_bytes_peak": 8378553344, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:12.113494Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5479320576, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 8378553344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113544003584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 570, + "segment_count_current_delta": 0, + "segment_count_peak": 570, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T141013452080Z.pickle", + "timestamp_utc": "2026-08-01T14:10:13.556319Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T141013452080Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T141013452080Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7bd8ccaf271093a21af0e09a289463e8f435d6c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T141013452080Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caba4abec1d5bcf81b102f95fc532a6205633f446ab0c7bce104519158bf0301 +size 6000873 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T141014609781Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T141014609781Z.json new file mode 100644 index 0000000000000000000000000000000000000000..43ba0b91b1e268011f48b8f325ecf659c99f28ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T141014609781Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110733676544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5483514880, + "inactive_split_bytes_peak": 8378553344, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:14.609368Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110733676544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5479320576, + "inactive_split_bytes_peak": 8378553344, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:13.568760Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5483514880, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8378553344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113544003584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 570, + "segment_count_current_delta": 0, + "segment_count_peak": 570, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T141014609781Z.pickle", + "timestamp_utc": "2026-08-01T14:10:14.711397Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T141014609781Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T141014609781Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dde799deccc025b0feaaf67f06cdafea6db7c862 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T141014609781Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00326467b521932a702bd1ee5842b4b9a6adad50759c8b9728afa20ca5c536b5 +size 5632401 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T141015757572Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T141015757572Z.json new file mode 100644 index 0000000000000000000000000000000000000000..57b1529a3444cda54c709c789cb60c7de373b301 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T141015757572Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110733676544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5718395904, + "inactive_split_bytes_peak": 8378553344, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:15.757149Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110733676544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5483514880, + "inactive_split_bytes_peak": 8378553344, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:14.723653Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5718395904, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 8378553344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113544003584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 570, + "segment_count_current_delta": 0, + "segment_count_peak": 570, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T141015757572Z.pickle", + "timestamp_utc": "2026-08-01T14:10:15.857088Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T141015757572Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T141015757572Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e2295cf7166a3ba8c89026fa364bf311d6f08d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T141015757572Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc8ca8d91da613a890ac4c5bb11d5e2a035e604a562827d7ffcfb9f6fcbe5064 +size 5633954 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T141016912923Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T141016912923Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9e34299ddb78fc2c39ebb333fb9c3c2f222246c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T141016912923Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110733676544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5716298752, + "inactive_split_bytes_peak": 8378553344, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:16.912499Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110733676544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5718395904, + "inactive_split_bytes_peak": 8378553344, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:15.869301Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5716298752, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8378553344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113544003584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 570, + "segment_count_current_delta": 0, + "segment_count_peak": 570, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T141016912923Z.pickle", + "timestamp_utc": "2026-08-01T14:10:17.014977Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T141016912923Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T141016912923Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e8eacf9318f594477255a94c4aabfb78f2cd977f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T141016912923Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e774eb7792b3052aed00d74e7abf033424d5d9fbc2199c9ba666e1a38046b283 +size 5632423 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T141018065834Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T141018065834Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ebd941781045bcc209b831ea1e138080433129f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T141018065834Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110733676544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5483514880, + "inactive_split_bytes_peak": 8378553344, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:18.065372Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110733676544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5716298752, + "inactive_split_bytes_peak": 8378553344, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:17.027526Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5483514880, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 8378553344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113544003584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 570, + "segment_count_current_delta": 0, + "segment_count_peak": 570, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T141018065834Z.pickle", + "timestamp_utc": "2026-08-01T14:10:18.166231Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T141018065834Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T141018065834Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c9e8eb76bb01850a328632d63d54378903003319 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T141018065834Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbe47bce028f26b18bad6104b36572bc6815ac2f98684183b23f2e1bc9c0c987 +size 5630880 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T141115593710Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T141115593710Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ed369b756da0a00a29ffec5b662898f09cd3abf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T141115593710Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110733676544, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5211196416, + "inactive_split_bytes_peak": 8378553344, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:11:15.593255Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110733676544, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110733676544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110733676544, + "device": 0, + "inactive_split_bytes_current": 5483514880, + "inactive_split_bytes_peak": 8378553344, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_peak": 113544003584, + "segment_count_current": 570, + "segment_count_peak": 570, + "timestamp_utc": "2026-08-01T14:10:18.178537Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5211196416, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 8378553344, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 113544003584, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113544003584, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 570, + "segment_count_current_delta": 0, + "segment_count_peak": 570, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T141115593710Z.pickle", + "timestamp_utc": "2026-08-01T14:11:15.815021Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T141115593710Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T141115593710Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ec7f00cc6e2d18b5e988b058a3fdb77f28048669 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T141115593710Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f3a84155af628d67da32ea9abea1f79e5aa8247944d7f596729c413c3cce2f1 +size 10227211 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T141008516823Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T141008516823Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ce860d6cad277cb1347beb24cc445783f302534 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T141008516823Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 87096555008, + "active_bytes_peak_delta": 51630218752, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 87096555008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 87096555008, + "device": 1, + "inactive_split_bytes_current": 5544332288, + "inactive_split_bytes_peak": 9112916992, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 96099893248, + "reserved_bytes_peak": 96099893248, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T14:10:08.516212Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 87096555008, + "allocated_bytes_peak_delta": 51630218752, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:09:01.305186Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5544332288, + "inactive_split_bytes_current_delta": 5411513344, + "inactive_split_bytes_peak": 9112916992, + "inactive_split_bytes_peak_delta": 8952530944, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 96099893248, + "reserved_bytes_current_delta": 56855887872, + "reserved_bytes_peak": 96099893248, + "reserved_bytes_peak_delta": 56855887872, + "segment_count_current": 496, + "segment_count_current_delta": 425, + "segment_count_peak": 496, + "segment_count_peak_delta": 425, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T141008516823Z.pickle", + "timestamp_utc": "2026-08-01T14:10:08.627729Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T141008516823Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T141008516823Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ffaa1433027e94953802fbfe1cefb5a06dad35a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T141008516823Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6ef1262e5b93e2d2014fffa453ffed4f66aa322973d05d77e77c55b71a83e7a +size 5395214 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T141009707983Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T141009707983Z.json new file mode 100644 index 0000000000000000000000000000000000000000..caa6c11de5a4246b7d34dfad1c1e1a8b1c695eaf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T141009707983Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110727492608, + "active_bytes_peak_delta": 23630937600, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110727492608, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110727492608, + "device": 1, + "inactive_split_bytes_current": 5328325632, + "inactive_split_bytes_peak": 9112916992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:09.707571Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110727492608, + "allocated_bytes_peak_delta": 23630937600, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 87096555008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 87096555008, + "device": 1, + "inactive_split_bytes_current": 5544332288, + "inactive_split_bytes_peak": 9112916992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 96099893248, + "reserved_bytes_peak": 96099893248, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T14:10:08.644460Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5328325632, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 9112916992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 17377001472, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 17377001472, + "segment_count_current": 566, + "segment_count_current_delta": 70, + "segment_count_peak": 566, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T141009707983Z.pickle", + "timestamp_utc": "2026-08-01T14:10:09.819678Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T141009707983Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T141009707983Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d9895a0967c83d8dde1009fa796d2faf135c7dd2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T141009707983Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fbd0175ce55f968abae7a6b3900ec50c31a040c85bbf9f7e25a691d30bfe393 +size 5625934 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T141010846058Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T141010846058Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6dedf40edddfb13ab74d3b7f80d79c796bc95b34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T141010846058Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 774144, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5324131328, + "inactive_split_bytes_peak": 9112916992, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:10.845656Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 774144, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110727492608, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110727492608, + "device": 1, + "inactive_split_bytes_current": 5328325632, + "inactive_split_bytes_peak": 9112916992, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:09.832644Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5324131328, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 9112916992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T141010846058Z.pickle", + "timestamp_utc": "2026-08-01T14:10:10.945159Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T141010846058Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T141010846058Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6871dd302871920249a19244f392bdde04d0afcc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T141010846058Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b600656d90aa4a4e4968d66547d40a571f2caf300fcf082e8cdbcb58c095eb7c +size 5623088 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T141011997623Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T141011997623Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3203f416884ae559df7ae476f959e5ef12c3722a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T141011997623Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_peak": 9112916992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:11.997230Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5324131328, + "inactive_split_bytes_peak": 9112916992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:10.957299Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 9112916992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T141011997623Z.pickle", + "timestamp_utc": "2026-08-01T14:10:12.100649Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T141011997623Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T141011997623Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..687202005b928b3988c19b6279b4d96a14ce5f7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T141011997623Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b12e0335d46abf64c680eb9bcd7b11ce78e1ad3de12be4e815902958eef5568d +size 5624334 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T141013452214Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T141013452214Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e75a7b636d43831a3bbcbe11c38fea7c1ea8946 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T141013452214Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5315742720, + "inactive_split_bytes_peak": 9112916992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:13.451799Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_peak": 9112916992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:12.114950Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5315742720, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 9112916992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T141013452214Z.pickle", + "timestamp_utc": "2026-08-01T14:10:13.554751Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T141013452214Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T141013452214Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b5b807283f26148aa3c0baf74fb604be4c43864b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T141013452214Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9682727a2d26b0d5a6d043f0bfb8b3d12ebb2457dfd5dc32bbcad078ff594b1f +size 5622834 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T141014609862Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T141014609862Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ce922dc328deceae93fa1f9ddfdd2867678fd256 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T141014609862Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9112916992, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:14.609442Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5315742720, + "inactive_split_bytes_peak": 9112916992, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:13.567503Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 9112916992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T141014609862Z.pickle", + "timestamp_utc": "2026-08-01T14:10:14.710704Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T141014609862Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T141014609862Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0fa9b79cbf6cac5e15e298ac4780902af8c86ef4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T141014609862Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c0822c36ac62cdc5d1dd190dd08f283d000d3b3aa2cac437b86534c8ace8681 +size 5624221 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T141015757596Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T141015757596Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea97d79518d982c75ca3d8cb2a28e197d7331d77 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T141015757596Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_peak": 9112916992, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:15.757184Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9112916992, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:14.722822Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 9112916992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T141015757596Z.pickle", + "timestamp_utc": "2026-08-01T14:10:15.852930Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T141015757596Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T141015757596Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5f473f928c240d628db0ecb78d6270e66c765f35 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T141015757596Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1502fc177e7182b28099aac13c179daece1a1d001378b230cad7039699adb5f6 +size 5624262 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T141016912988Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T141016912988Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad6477d53eeb4456183a9a6c52f00616a54650ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T141016912988Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5317839872, + "inactive_split_bytes_peak": 9112916992, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:16.912566Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_peak": 9112916992, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:15.864668Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5317839872, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 9112916992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T141016912988Z.pickle", + "timestamp_utc": "2026-08-01T14:10:17.011080Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T141016912988Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T141016912988Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..795c626a081fa54aa01af6583325d817e379159f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T141016912988Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4f95ccf776d061f902233ef4c1f2eb82879c86500b1c812ff0af0d42017c154 +size 5624196 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T141018065749Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T141018065749Z.json new file mode 100644 index 0000000000000000000000000000000000000000..974659b902034a29a2363e4aeee9471447806aee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T141018065749Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9112916992, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:18.065345Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5317839872, + "inactive_split_bytes_peak": 9112916992, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:17.023298Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 9112916992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T141018065749Z.pickle", + "timestamp_utc": "2026-08-01T14:10:18.158491Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T141018065749Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T141018065749Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0fae8e51bdfcc636d079ec49b441b3a6c3232ade --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T141018065749Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50550dd029a8e9e727e86f58d85bbda0b4b938c1900daa7f8509eb65dd9bc911 +size 5624289 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T141115593723Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T141115593723Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d41bda6c3a1c4c681cfca19d662c2034265b6758 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T141115593723Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5047618560, + "inactive_split_bytes_peak": 9112916992, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:11:15.593259Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 1, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9112916992, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:18.169934Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5047618560, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 9112916992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T141115593723Z.pickle", + "timestamp_utc": "2026-08-01T14:11:15.798143Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T141115593723Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T141115593723Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e631ce8d707273ffd24871da806bf2b26b7c61a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T141115593723Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c44ddc425dc12b93239481782908fab4c8638714ee5d11c866d41b243d29b8a5 +size 9552324 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T141008516749Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T141008516749Z.json new file mode 100644 index 0000000000000000000000000000000000000000..28dbefdd679dc716c2f5a235e20450ab28503d49 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T141008516749Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 87096555008, + "active_bytes_peak_delta": 51630218752, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 87096555008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 87096555008, + "device": 2, + "inactive_split_bytes_current": 5544332288, + "inactive_split_bytes_peak": 9110811648, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 96569655296, + "reserved_bytes_peak": 96569655296, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T14:10:08.516182Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 87096555008, + "allocated_bytes_peak_delta": 51630218752, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:09:01.298183Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5544332288, + "inactive_split_bytes_current_delta": 5411513344, + "inactive_split_bytes_peak": 9110811648, + "inactive_split_bytes_peak_delta": 8950425600, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 96569655296, + "reserved_bytes_current_delta": 57325649920, + "reserved_bytes_peak": 96569655296, + "reserved_bytes_peak_delta": 57325649920, + "segment_count_current": 498, + "segment_count_current_delta": 427, + "segment_count_peak": 498, + "segment_count_peak_delta": 427, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T141008516749Z.pickle", + "timestamp_utc": "2026-08-01T14:10:08.629010Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T141008516749Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T141008516749Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9700209b1097e764951e13f0a712c3bfbd27cb74 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T141008516749Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:484f893e7cdd47b7887a2000f7e4205e3e81cdad0957974753f809bbf1785c74 +size 5395546 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T141009707908Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T141009707908Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e5ccc0a8194604217faec6d9422233490973ed1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T141009707908Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 23631743488, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5326228480, + "inactive_split_bytes_peak": 9110811648, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:09.707507Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 23631743488, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 87096555008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 87096555008, + "device": 2, + "inactive_split_bytes_current": 5544332288, + "inactive_split_bytes_peak": 9110811648, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 96569655296, + "reserved_bytes_peak": 96569655296, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T14:10:08.648156Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5326228480, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 9110811648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 17435721728, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 17435721728, + "segment_count_current": 569, + "segment_count_current_delta": 71, + "segment_count_peak": 569, + "segment_count_peak_delta": 71, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T141009707908Z.pickle", + "timestamp_utc": "2026-08-01T14:10:09.818311Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T141009707908Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T141009707908Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..980e8f53523b10c5dd9cf9801e328d513b46ef8b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T141009707908Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cb6ed047d7c5f2713762b35dbb9bfe60df1312387a1fcb1697d055d761878cd +size 5626500 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T141010846056Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T141010846056Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5dd939a5e0e0294ac51a8918915753e70c2541d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T141010846056Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5324131328, + "inactive_split_bytes_peak": 9110811648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:10.845663Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5326228480, + "inactive_split_bytes_peak": 9110811648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:09.832392Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5324131328, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 9110811648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T141010846056Z.pickle", + "timestamp_utc": "2026-08-01T14:10:10.956173Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T141010846056Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T141010846056Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..50eeff9d93fa63b320c1ce848b08c984eeb8a75b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T141010846056Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56378be0d3ae678f061cfc74e100e9b74418a43910ae6dbeb1c860719700e742 +size 5623655 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T141011997576Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T141011997576Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f77b2c99f7c0a825a3726f678461ca11455f6c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T141011997576Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_peak": 9110811648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:11.997155Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5324131328, + "inactive_split_bytes_peak": 9110811648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:10.972148Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 9110811648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T141011997576Z.pickle", + "timestamp_utc": "2026-08-01T14:10:12.098101Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T141011997576Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T141011997576Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..83eb29a9b521edf4515cc288ee0502cf69700bb3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T141011997576Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3ddde7af5e0c48272eefedabdbe26a9d82be07fad8c15fa6a689075ce8eca7b +size 5624935 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T141013452124Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T141013452124Z.json new file mode 100644 index 0000000000000000000000000000000000000000..82d80c937fd1e325f097b6d40c3a9e6ba15457ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T141013452124Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5315742720, + "inactive_split_bytes_peak": 9110811648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:13.451714Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_peak": 9110811648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:12.111206Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5315742720, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 9110811648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T141013452124Z.pickle", + "timestamp_utc": "2026-08-01T14:10:13.556830Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T141013452124Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T141013452124Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e0a705e8588cac1396430d0c098d77cad7750f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T141013452124Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dd9f40a4f065ee2fc12e6afd4a6d6f07c3eda4d0c171d015ec9455576a637d9 +size 5623460 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T141014609792Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T141014609792Z.json new file mode 100644 index 0000000000000000000000000000000000000000..71e4b951b87ee0b57fda581b6c398b8bbbe0554a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T141014609792Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_peak": 9110811648, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:14.609389Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5315742720, + "inactive_split_bytes_peak": 9110811648, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:13.570305Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 9110811648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T141014609792Z.pickle", + "timestamp_utc": "2026-08-01T14:10:14.708474Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T141014609792Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T141014609792Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b68fcf59ab1a198524f2bf97631f3dfd768c7fa7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T141014609792Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:766c634f19b441883f0e37498282c583728e16c3d099bed16c3e59f388f04bee +size 5624871 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T141015757511Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T141015757511Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bb38c332a64c472d5baf2e09245521962950ec70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T141015757511Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9110811648, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:15.757103Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_peak": 9110811648, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:14.721469Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 9110811648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T141015757511Z.pickle", + "timestamp_utc": "2026-08-01T14:10:15.856989Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T141015757511Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T141015757511Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1e1223f5260f732e5e58f73d4445e7ce433c8e5e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T141015757511Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47ef3c4609c31e3da3688918c9215a9814f1f41e9f87594ec06b39f8729c9576 +size 5624852 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T141016912930Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T141016912930Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a67366cc791d3cc9b163e28330350d10ee19dd9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T141016912930Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5317839872, + "inactive_split_bytes_peak": 9110811648, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:16.912507Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9110811648, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:15.870190Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5317839872, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 9110811648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T141016912930Z.pickle", + "timestamp_utc": "2026-08-01T14:10:17.012441Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T141016912930Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T141016912930Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..be9ed2a91a75cbd3ea091b540367089ac2f7b21c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T141016912930Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3466f921f1c175672790dc77d25f0afe4b407861fd2814798b966b8974d0dd6 +size 5624692 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T141018065688Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T141018065688Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c7195c0459908294bdd8085467b1771c431e7091 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T141018065688Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9110811648, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:18.065284Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5317839872, + "inactive_split_bytes_peak": 9110811648, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:17.025354Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 9110811648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T141018065688Z.pickle", + "timestamp_utc": "2026-08-01T14:10:18.161219Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T141018065688Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T141018065688Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..15729335b849af496588bd6b96c76bc3d9d25fc6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T141018065688Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d06e72812417a5e7cdbcd918958ff74b4377585977628a1581ba2910b03ab76a +size 5624855 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T141115593718Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T141115593718Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d5d7900b5af8abb5514bd17e5e819508d6c8828 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T141115593718Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5047618560, + "inactive_split_bytes_peak": 9110811648, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:11:15.593246Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 2, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9110811648, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:18.173779Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5047618560, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 9110811648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T141115593718Z.pickle", + "timestamp_utc": "2026-08-01T14:11:15.796575Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T141115593718Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T141115593718Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb2b77b4648ce56374dbce0624b9acd713889bae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T141115593718Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9adeeec6fb9415b5bc28c9053fb231ade11c26f4039df8fc3f8691c06833d796 +size 9552758 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T141008516711Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T141008516711Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ff99e01db9dae5579f94ba8acf7d78eeb22f780e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T141008516711Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 87097444352, + "active_bytes_peak_delta": 51631108096, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 87097444352, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 87097444352, + "device": 3, + "inactive_split_bytes_current": 5779213312, + "inactive_split_bytes_peak": 8698385408, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 96569655296, + "reserved_bytes_peak": 96569655296, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T14:10:08.516133Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 87097444352, + "allocated_bytes_peak_delta": 51631108096, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:09:01.297348Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5779213312, + "inactive_split_bytes_current_delta": 5646394368, + "inactive_split_bytes_peak": 8698385408, + "inactive_split_bytes_peak_delta": 8537999360, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 96569655296, + "reserved_bytes_current_delta": 57325649920, + "reserved_bytes_peak": 96569655296, + "reserved_bytes_peak_delta": 57325649920, + "segment_count_current": 498, + "segment_count_current_delta": 427, + "segment_count_peak": 498, + "segment_count_peak_delta": 427, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T141008516711Z.pickle", + "timestamp_utc": "2026-08-01T14:10:08.637783Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T141008516711Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T141008516711Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d9f7c95b46032cd1f5ba251d005ab33774bfac6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T141008516711Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7257d441b46a9223a756e53ad7e07c74939112acbb2439739ca21d1ab5df7e5 +size 5395606 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T141009707923Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T141009707923Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d41bb495872e360b3f1b29ef142a2d35e20a0dbe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T141009707923Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 23630854144, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5563206656, + "inactive_split_bytes_peak": 8698385408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:09.707521Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 23630854144, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 87097444352, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 87097444352, + "device": 3, + "inactive_split_bytes_current": 5779213312, + "inactive_split_bytes_peak": 8698385408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 96569655296, + "reserved_bytes_peak": 96569655296, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T14:10:08.657043Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5563206656, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 8698385408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 17435721728, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 17435721728, + "segment_count_current": 569, + "segment_count_current_delta": 71, + "segment_count_peak": 569, + "segment_count_peak_delta": 71, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T141009707923Z.pickle", + "timestamp_utc": "2026-08-01T14:10:09.820280Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T141009707923Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T141009707923Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b8b27adfb631226247f29d1cf3e6daffc45e554 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T141009707923Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fba2afbba74cf602cdf56f62db86193797f58a49ecc9b8744582f3b3f16b9799 +size 5626684 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T141010846055Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T141010846055Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f077bcdccb5536da5c2b4f44882f892213edc063 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T141010846055Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5559012352, + "inactive_split_bytes_peak": 8698385408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:10.845656Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5563206656, + "inactive_split_bytes_peak": 8698385408, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:09.833988Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5559012352, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 8698385408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T141010846055Z.pickle", + "timestamp_utc": "2026-08-01T14:10:10.948089Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T141010846055Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T141010846055Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eaf2ade4196b587be85a650b5c0c6b3e2fd6b1a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T141010846055Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f773bb68de623be5457393b2aa240d1fde1a41debce0773a2117456a2b0b0ed +size 5623881 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T141011997584Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T141011997584Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8804a3d63b214c28aeb6af290d79ef13e9143216 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T141011997584Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_peak": 8698385408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:11.997182Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5559012352, + "inactive_split_bytes_peak": 8698385408, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:10.961139Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 8698385408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T141011997584Z.pickle", + "timestamp_utc": "2026-08-01T14:10:12.100268Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T141011997584Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T141011997584Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d966921b85f194229ab26c951f25ad9a7457abc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T141011997584Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84c37023bd5ac7f7e3200a0a27dfd6dfd7de28ae0e505a25a9f0a6234e496cf8 +size 5624918 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T141013452101Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T141013452101Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f887a0599697917f0a11377b97e0da5e0ad8f5ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T141013452101Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5550623744, + "inactive_split_bytes_peak": 8698385408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:13.451691Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_peak": 8698385408, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:12.113229Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5550623744, + "inactive_split_bytes_current_delta": 228589568, + "inactive_split_bytes_peak": 8698385408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T141013452101Z.pickle", + "timestamp_utc": "2026-08-01T14:10:13.557685Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T141013452101Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T141013452101Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..246864cb294b2bcbfc28646b324bb0a2d66c8673 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T141013452101Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71a047322af6a30295a870cdd1b93f3b1feaa52b4918ad7bc2b1729d8b39c191 +size 5623414 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T141014609786Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T141014609786Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c5568ec85b8794dcb0c04732a3b4ff2b9c86406c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T141014609786Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_peak": 8698385408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:14.609379Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5550623744, + "inactive_split_bytes_peak": 8698385408, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:13.571002Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8698385408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T141014609786Z.pickle", + "timestamp_utc": "2026-08-01T14:10:14.709350Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T141014609786Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T141014609786Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..09dc3588e83da3c1aaa994775047ca720dc0ebf8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T141014609786Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43078508c2fab5d3d4931ebe9fcdb555d15b190c5b197dde4b340814589ec21b +size 5624739 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T141015757518Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T141015757518Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3a6efbc90d1150f47bb8ca73d8a9866b6996c044 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T141015757518Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_peak": 8698385408, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:15.757105Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_peak": 8698385408, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:14.722116Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8698385408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T141015757518Z.pickle", + "timestamp_utc": "2026-08-01T14:10:15.857365Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T141015757518Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T141015757518Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..42539369dca16bcbf8abdb2cb30a813bb7d3d780 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T141015757518Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60b61349ac2a3d09792fd5689085053fc145c0bb9af2fc54a5c9d14aea15871a +size 5624806 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T141016912927Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T141016912927Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8fbbada5e84d39545bdb754e1b6fcb5af120e10a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T141016912927Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5317839872, + "inactive_split_bytes_peak": 8698385408, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:16.912511Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5554818048, + "inactive_split_bytes_peak": 8698385408, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:15.870135Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5317839872, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 8698385408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T141016912927Z.pickle", + "timestamp_utc": "2026-08-01T14:10:17.013254Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T141016912927Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T141016912927Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ffbd7894c586770a04bbfa9d1fe07d8f56dc688c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T141016912927Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b41a105387c9667f5dc7ff4e1e8584d89e398a09494f1969ac9be0d291c78f90 +size 5624678 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T141018065717Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T141018065717Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7eb797e9e3f4ab0c99b12efcf5afb8e93130b452 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T141018065717Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 8698385408, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:18.065293Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5317839872, + "inactive_split_bytes_peak": 8698385408, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:17.026030Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 8698385408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T141018065717Z.pickle", + "timestamp_utc": "2026-08-01T14:10:18.163078Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T141018065717Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T141018065717Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..58eb8573fba8c471fde7f484baab0002fe3b0b5c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T141018065717Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eedcd42e6582bf837bf13ebb804243278f3ec2e1439869118998534f5c113322 +size 5624788 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T141115593718Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T141115593718Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ff107730549a2a15f7f4cecc2fa5bca4814535d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T141115593718Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110728298496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5047618560, + "inactive_split_bytes_peak": 8698385408, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:11:15.593246Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110728298496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728298496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728298496, + "device": 3, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 8698385408, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 114005377024, + "reserved_bytes_peak": 114005377024, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:18.175945Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5047618560, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 8698385408, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 114005377024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114005377024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T141115593718Z.pickle", + "timestamp_utc": "2026-08-01T14:11:15.796889Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T141115593718Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T141115593718Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b046b4c6327a094d4ae49c82ea2db0e1897c13b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T141115593718Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9538a6f0fe4335d7a74f2b8d6641817231435a70c3337da7187405ce8de91cb4 +size 9552833 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T141008516686Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T141008516686Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd4e8a612f88e366a3a2f277027ee0f42da1e12d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T141008516686Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854690816, + "active_bytes_current_delta": 31622793216, + "active_bytes_peak": 87102234624, + "active_bytes_peak_delta": 51635456000, + "after": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 87102234624, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 87102234624, + "device": 4, + "inactive_split_bytes_current": 5176188928, + "inactive_split_bytes_peak": 7710994432, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 96919879680, + "reserved_bytes_peak": 96919879680, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:10:08.516263Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854690816, + "allocated_bytes_current_delta": 31622793216, + "allocated_bytes_peak": 87102234624, + "allocated_bytes_peak_delta": 51635456000, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:09:01.297189Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5176188928, + "inactive_split_bytes_current_delta": 5043812352, + "inactive_split_bytes_peak": 7710994432, + "inactive_split_bytes_peak_delta": 7557584896, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 96919879680, + "reserved_bytes_current_delta": 57675874304, + "reserved_bytes_peak": 96919879680, + "reserved_bytes_peak_delta": 57675874304, + "segment_count_current": 500, + "segment_count_current_delta": 429, + "segment_count_peak": 500, + "segment_count_peak_delta": 429, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T141008516686Z.pickle", + "timestamp_utc": "2026-08-01T14:10:08.628597Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T141008516686Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T141008516686Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..475b9f27924daf41b7136d818beb583f09b80176 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T141008516686Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e43a37e2ba581076bae90ca3587a3b27c3161128dea5ef5493ba88dcd5aabd7 +size 5405313 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T141009707925Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T141009707925Z.json new file mode 100644 index 0000000000000000000000000000000000000000..40842e2619d860467a7d3d411e0d6f095085dc08 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T141009707925Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854690816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110734848000, + "active_bytes_peak_delta": 23632613376, + "after": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110734848000, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110734848000, + "device": 4, + "inactive_split_bytes_current": 4958085120, + "inactive_split_bytes_peak": 7710994432, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:09.707487Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854690816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110734848000, + "allocated_bytes_peak_delta": 23632613376, + "before": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 87102234624, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 87102234624, + "device": 4, + "inactive_split_bytes_current": 5176188928, + "inactive_split_bytes_peak": 7710994432, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 96919879680, + "reserved_bytes_peak": 96919879680, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:10:08.647454Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4958085120, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 7710994432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 114179440640, + "reserved_bytes_current_delta": 17259560960, + "reserved_bytes_peak": 114179440640, + "reserved_bytes_peak_delta": 17259560960, + "segment_count_current": 569, + "segment_count_current_delta": 69, + "segment_count_peak": 569, + "segment_count_peak_delta": 69, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T141009707925Z.pickle", + "timestamp_utc": "2026-08-01T14:10:09.820734Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T141009707925Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T141009707925Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d7a2429219951d21f89d4cae7b8e9bff5f46a9b4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T141009707925Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:489fa644c41e4b12fcd39d3c2c0390fff87c5fa4f160ff35a232fbb934ae95d7 +size 5633489 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T141010845983Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T141010845983Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aab2c0693e9a0831bfce2eb43476c48c960a09de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T141010845983Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854690816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110736945152, + "active_bytes_peak_delta": 2097152, + "after": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4955987968, + "inactive_split_bytes_peak": 8832103424, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:10.845571Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854690816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110736945152, + "allocated_bytes_peak_delta": 2097152, + "before": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110734848000, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110734848000, + "device": 4, + "inactive_split_bytes_current": 4958085120, + "inactive_split_bytes_peak": 7710994432, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:09.835833Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4955987968, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8832103424, + "inactive_split_bytes_peak_delta": 1121108992, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 114179440640, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114179440640, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T141010845983Z.pickle", + "timestamp_utc": "2026-08-01T14:10:10.950552Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T141010845983Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T141010845983Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..685a081b1ce2aa179140e43ecd0b7310887e453b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T141010845983Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57fa4483df1ea2e66ed4de70727b603a3a3975300fd3b1f23c52de6393f4d721 +size 5627624 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T141011997552Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T141011997552Z.json new file mode 100644 index 0000000000000000000000000000000000000000..062a03673d843ea4fa749a22520b45c7c2bc308e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T141011997552Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854690816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110736945152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4953890816, + "inactive_split_bytes_peak": 8832103424, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:11.997132Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854690816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110736945152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4955987968, + "inactive_split_bytes_peak": 8832103424, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:10.965217Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4953890816, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8832103424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 114179440640, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114179440640, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T141011997552Z.pickle", + "timestamp_utc": "2026-08-01T14:10:12.104624Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T141011997552Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T141011997552Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f88e9e8613eb5d3fa77c10260a1e326e70554d4e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T141011997552Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e002ae89fc2fc0999b2bd08486ce9fb98a46dcb7bd6ea70ff2810fea1382b26b +size 5632290 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T141013452081Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T141013452081Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f73ba30153b623bc615cb37a0fdf31cdf14d4dc5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T141013452081Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854690816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110736945152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4947599360, + "inactive_split_bytes_peak": 8832103424, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:13.451677Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854690816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110736945152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4953890816, + "inactive_split_bytes_peak": 8832103424, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:12.119341Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4947599360, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 8832103424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 114179440640, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114179440640, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T141013452081Z.pickle", + "timestamp_utc": "2026-08-01T14:10:13.557683Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T141013452081Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T141013452081Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf776ad886aaa90056e0413a65d9b7658a188e90 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T141013452081Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7a13345b6240a41f49c3d7bfe38d22bc0943ed817bc10bdc2c7ce4227851331 +size 5628757 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T141014609766Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T141014609766Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf4e3ae7597a07c5e8126ab782d1cc57d2271786 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T141014609766Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854690816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110736945152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4719009792, + "inactive_split_bytes_peak": 8832103424, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:14.609361Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854690816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110736945152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4947599360, + "inactive_split_bytes_peak": 8832103424, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:13.572273Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4719009792, + "inactive_split_bytes_current_delta": -228589568, + "inactive_split_bytes_peak": 8832103424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 114179440640, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114179440640, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T141014609766Z.pickle", + "timestamp_utc": "2026-08-01T14:10:14.712905Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T141014609766Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T141014609766Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7579129c950854405584fad022b5b8fc008d67d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T141014609766Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26053213be2e303b127a1206feb3ff14c94c8be59ccbdd78986ca8020ca26493 +size 5632091 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T141015757478Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T141015757478Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ed147816ebfb7e9272ee981676306ef6853453e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T141015757478Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854690816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110736945152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4719009792, + "inactive_split_bytes_peak": 8832103424, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:15.757075Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854690816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110736945152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4719009792, + "inactive_split_bytes_peak": 8832103424, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:14.727391Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4719009792, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8832103424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 114179440640, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114179440640, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T141015757478Z.pickle", + "timestamp_utc": "2026-08-01T14:10:15.859831Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T141015757478Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T141015757478Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..440b630a90250eab85cb4c6c9cc1d7ede2835683 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T141015757478Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62e49df9ff1c88c68d34b32528a55f11cd2406528099aeaec579c8b1dbb82491 +size 5631134 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T141016912926Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T141016912926Z.json new file mode 100644 index 0000000000000000000000000000000000000000..af50ce5dda27f33fbb4e1a4ec9e0a167c7e37d66 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T141016912926Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854690816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110736945152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4951793664, + "inactive_split_bytes_peak": 8832103424, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:16.912490Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854690816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110736945152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4719009792, + "inactive_split_bytes_peak": 8832103424, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:15.874181Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4951793664, + "inactive_split_bytes_current_delta": 232783872, + "inactive_split_bytes_peak": 8832103424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 114179440640, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114179440640, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T141016912926Z.pickle", + "timestamp_utc": "2026-08-01T14:10:17.020767Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T141016912926Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T141016912926Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d890c98c1f5853c65a0676c200e11cc8f04d8832 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T141016912926Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ee708f62637b9846e7996b64599fab608a21b4a1da93010af16a174be8a4d01 +size 5631749 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T141018065683Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T141018065683Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a2920b6336938f9661b6f3e20656a0d4c314a1ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T141018065683Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854690816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110736945152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4719009792, + "inactive_split_bytes_peak": 8832103424, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:18.065265Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854690816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110736945152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4951793664, + "inactive_split_bytes_peak": 8832103424, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:17.042659Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4719009792, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 8832103424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 114179440640, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114179440640, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T141018065683Z.pickle", + "timestamp_utc": "2026-08-01T14:10:18.167536Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T141018065683Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T141018065683Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e730ea6954a87d16f9a725f3142c48f88fd2295 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T141018065683Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c60345bb63166746e97bfce4fee680a4e27ee02d4d2553951700149ba1209921 +size 5631821 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T141115593761Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T141115593761Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e1fc9b808fbf3b4505b66d694d825d930f714971 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T141115593761Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804047872, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110736945152, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66804047872, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66804047872, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4444594176, + "inactive_split_bytes_peak": 8832103424, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:11:15.593265Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804047872, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110736945152, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854690816, + "active_bytes_peak": 110736945152, + "allocated_bytes_current": 66854690816, + "allocated_bytes_peak": 110736945152, + "device": 4, + "inactive_split_bytes_current": 4719009792, + "inactive_split_bytes_peak": 8832103424, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 114179440640, + "reserved_bytes_peak": 114179440640, + "segment_count_current": 569, + "segment_count_peak": 569, + "timestamp_utc": "2026-08-01T14:10:18.181914Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4444594176, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 8832103424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 114179440640, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 114179440640, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 569, + "segment_count_current_delta": 0, + "segment_count_peak": 569, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T141115593761Z.pickle", + "timestamp_utc": "2026-08-01T14:11:15.791290Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T141115593761Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T141115593761Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2b5500f6061236d6f3ca0a28f185b826923c5d55 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T141115593761Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18e0501528205f794b69b19b2c4d0a461f67f0fd4843ec4f410c482f2b35fa82 +size 10223482 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T141008516679Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T141008516679Z.json new file mode 100644 index 0000000000000000000000000000000000000000..30c0b689204a0f6d8bc4ab904bb58c3361d868a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T141008516679Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 87096555008, + "active_bytes_peak_delta": 51630218752, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 87096555008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 87096555008, + "device": 5, + "inactive_split_bytes_current": 5544332288, + "inactive_split_bytes_peak": 9230275584, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 96569655296, + "reserved_bytes_peak": 96569655296, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T14:10:08.516113Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 87096555008, + "allocated_bytes_peak_delta": 51630218752, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:09:01.297193Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5544332288, + "inactive_split_bytes_current_delta": 5411513344, + "inactive_split_bytes_peak": 9230275584, + "inactive_split_bytes_peak_delta": 9069889536, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 96569655296, + "reserved_bytes_current_delta": 57325649920, + "reserved_bytes_peak": 96569655296, + "reserved_bytes_peak_delta": 57325649920, + "segment_count_current": 498, + "segment_count_current_delta": 427, + "segment_count_peak": 498, + "segment_count_peak_delta": 427, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T141008516679Z.pickle", + "timestamp_utc": "2026-08-01T14:10:08.626018Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T141008516679Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T141008516679Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3ded86c91d57413016df3e5833efca93b65d1d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T141008516679Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7a1c7e297a2a56bdc8ac82828c317c7a00002660a5bd75b9a554b010307ca4d +size 5395553 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T141009707910Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T141009707910Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b305c84f794603b41eaa4655ae14cfa64635cd81 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T141009707910Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110727492608, + "active_bytes_peak_delta": 23630937600, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110727492608, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110727492608, + "device": 5, + "inactive_split_bytes_current": 5326228480, + "inactive_split_bytes_peak": 9230275584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:09.707502Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110727492608, + "allocated_bytes_peak_delta": 23630937600, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 87096555008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 87096555008, + "device": 5, + "inactive_split_bytes_current": 5544332288, + "inactive_split_bytes_peak": 9230275584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 96569655296, + "reserved_bytes_peak": 96569655296, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T14:10:08.643641Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5326228480, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 9230275584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 17377001472, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 17377001472, + "segment_count_current": 568, + "segment_count_current_delta": 70, + "segment_count_peak": 568, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T141009707910Z.pickle", + "timestamp_utc": "2026-08-01T14:10:09.819257Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T141009707910Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T141009707910Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..720a7c6abd0bbbde566ec60939eb9adb47d0d6f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T141009707910Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21dfa5b7bf59e1b2b00aa2e4263c314c84dcaff0ad3ca27f89ec0c5685b8d8d1 +size 5626815 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T141010845988Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T141010845988Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f7a49d5002cbe3fc56fc621d99ebf6d3d1162a2d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T141010845988Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 774144, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5324131328, + "inactive_split_bytes_peak": 9230275584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:10.845583Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 774144, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110727492608, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110727492608, + "device": 5, + "inactive_split_bytes_current": 5326228480, + "inactive_split_bytes_peak": 9230275584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:09.834395Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5324131328, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 9230275584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T141010845988Z.pickle", + "timestamp_utc": "2026-08-01T14:10:10.949915Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T141010845988Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T141010845988Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ac65bf623e758f49c5e55559f85e160294435550 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T141010845988Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c3a5b5012cc28537012fa46b4889cfac994aa6c0003ea5e3435aa05e975b9c9 +size 5623682 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T141011997528Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T141011997528Z.json new file mode 100644 index 0000000000000000000000000000000000000000..543ade21244996fa67dc09a451662cf138089867 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T141011997528Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_peak": 9230275584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:11.997127Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5324131328, + "inactive_split_bytes_peak": 9230275584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:10.964582Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 9230275584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T141011997528Z.pickle", + "timestamp_utc": "2026-08-01T14:10:12.101239Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T141011997528Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T141011997528Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9340662bb481df89b356fd1c4de0b14c495a3314 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T141011997528Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b93e51d4335facd87acee2f611cbbd303164550a4bce93832b0f436d65d0165f +size 5624976 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T141013452079Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T141013452079Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db89a0f49f55bb00a2c655c0833a6a577fb7fa9a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T141013452079Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5315742720, + "inactive_split_bytes_peak": 9230275584, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:13.451672Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_peak": 9230275584, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:12.115753Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5315742720, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 9230275584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T141013452079Z.pickle", + "timestamp_utc": "2026-08-01T14:10:13.562389Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T141013452079Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T141013452079Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a6a88cef0ca5c118cce4c3f1287754317ef48b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T141013452079Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86f37ccd2dac8d2cc3661db4ab18f203c16c26208cbf35332fe6d628000c7da6 +size 5623498 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T141014609755Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T141014609755Z.json new file mode 100644 index 0000000000000000000000000000000000000000..79cb130d5f2fb091e0dabc141bc9ca0bf279b041 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T141014609755Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9230275584, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:14.609356Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5315742720, + "inactive_split_bytes_peak": 9230275584, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:13.579205Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 9230275584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T141014609755Z.pickle", + "timestamp_utc": "2026-08-01T14:10:14.709774Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T141014609755Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T141014609755Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..570e92f3ba568c1b2862279e34bd58d1e70a177c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T141014609755Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c46a60cc5fe32f82f6adfa81f6c9b541d05cbc3d120d21333e35c2e6cbeadb4f +size 5624820 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T141015757466Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T141015757466Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15653434ab33bebefe5207394fc2318d69872ff3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T141015757466Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9230275584, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:15.757072Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9230275584, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:14.723897Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 9230275584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T141015757466Z.pickle", + "timestamp_utc": "2026-08-01T14:10:15.861100Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T141015757466Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T141015757466Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7954e37a0985022fc3c6dbd23112c1f19a93cb67 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T141015757466Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d36da75fbea756b93e163e7209be9b73d630962b269b7510a9b094b2a48e42b +size 5624895 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T141016912927Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T141016912927Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e41d218320c0a15ab7448b26617a610c35eb8d88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T141016912927Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5552720896, + "inactive_split_bytes_peak": 9230275584, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:16.912499Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9230275584, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:15.875525Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5552720896, + "inactive_split_bytes_current_delta": 232783872, + "inactive_split_bytes_peak": 9230275584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T141016912927Z.pickle", + "timestamp_utc": "2026-08-01T14:10:17.014834Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T141016912927Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T141016912927Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6212c85f13fddc54f83f26cfc6f1a60aaa3574ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T141016912927Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bdb0fefe409e7a7477323e18715e3addcb260ea4d0c58cd6f6715616cd906e3 +size 5624715 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T141018065684Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T141018065684Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd4c65c9b20ca7ae00ca69b2371e9b2d6dbb7474 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T141018065684Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9230275584, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:18.065281Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5552720896, + "inactive_split_bytes_peak": 9230275584, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:17.029312Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 9230275584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T141018065684Z.pickle", + "timestamp_utc": "2026-08-01T14:10:18.166095Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T141018065684Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T141018065684Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..36758e6e69c98e6d666e24d198e14f6816dc44d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T141018065684Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:608965ab611f81dac3a13e71ece451cbc35d3cdb508839c27eece252dd6a9ce4 +size 5624720 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T141115593718Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T141115593718Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a321a9012a43b7b14d44c31b879849e6919ba7c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T141115593718Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5047618560, + "inactive_split_bytes_peak": 9230275584, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:11:15.593250Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 5, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 9230275584, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:18.180365Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5047618560, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 9230275584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T141115593718Z.pickle", + "timestamp_utc": "2026-08-01T14:11:15.790617Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T141115593718Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T141115593718Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..faaf81ed831689317d80857e191d479476f45992 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T141115593718Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f6fb9ffe925916607657f06e96208e8089a30e568054ded254965150d238816 +size 9552797 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T141008516729Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T141008516729Z.json new file mode 100644 index 0000000000000000000000000000000000000000..12eac9000eccb821b6623aa5ec18bbfaa8daf38d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T141008516729Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 87096555008, + "active_bytes_peak_delta": 51630218752, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 87096555008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 87096555008, + "device": 6, + "inactive_split_bytes_current": 5544332288, + "inactive_split_bytes_peak": 8760513536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 96099893248, + "reserved_bytes_peak": 96099893248, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T14:10:08.516153Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 87096555008, + "allocated_bytes_peak_delta": 51630218752, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:09:01.297926Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5544332288, + "inactive_split_bytes_current_delta": 5411513344, + "inactive_split_bytes_peak": 8760513536, + "inactive_split_bytes_peak_delta": 8600127488, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 96099893248, + "reserved_bytes_current_delta": 56855887872, + "reserved_bytes_peak": 96099893248, + "reserved_bytes_peak_delta": 56855887872, + "segment_count_current": 496, + "segment_count_current_delta": 425, + "segment_count_peak": 496, + "segment_count_peak_delta": 425, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T141008516729Z.pickle", + "timestamp_utc": "2026-08-01T14:10:08.629383Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T141008516729Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T141008516729Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3554da719424e3e0f351a7697a743a023de1a3e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T141008516729Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a29a11407c61035a987d723f8c3bac19a224a480d1ec3739a6b5a031615bf06 +size 5395213 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T141009707896Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T141009707896Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42dd49b316b0b16c003be301e4770bac40fb85f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T141009707896Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110727492608, + "active_bytes_peak_delta": 23630937600, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110727492608, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110727492608, + "device": 6, + "inactive_split_bytes_current": 5326228480, + "inactive_split_bytes_peak": 8760513536, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:09.707486Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110727492608, + "allocated_bytes_peak_delta": 23630937600, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 87096555008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 87096555008, + "device": 6, + "inactive_split_bytes_current": 5544332288, + "inactive_split_bytes_peak": 8760513536, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 96099893248, + "reserved_bytes_peak": 96099893248, + "segment_count_current": 496, + "segment_count_peak": 496, + "timestamp_utc": "2026-08-01T14:10:08.647158Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5326228480, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 8760513536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 17377001472, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 17377001472, + "segment_count_current": 566, + "segment_count_current_delta": 70, + "segment_count_peak": 566, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T141009707896Z.pickle", + "timestamp_utc": "2026-08-01T14:10:09.820147Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T141009707896Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T141009707896Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dd844b654aafd0c45c36b3d2a89b6a576ecd741c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T141009707896Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36cf20e9195f3a1512e0aa3e95cbc2f7f190c218be12be389e52a62800ba5ed7 +size 5625832 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T141010846056Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T141010846056Z.json new file mode 100644 index 0000000000000000000000000000000000000000..354b2d6ad99b9616675ccfe58655a5c7c546d3a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T141010846056Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 774144, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5324131328, + "inactive_split_bytes_peak": 8760513536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:10.845656Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 774144, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110727492608, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110727492608, + "device": 6, + "inactive_split_bytes_current": 5326228480, + "inactive_split_bytes_peak": 8760513536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:09.835394Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5324131328, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8760513536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T141010846056Z.pickle", + "timestamp_utc": "2026-08-01T14:10:10.951553Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T141010846056Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T141010846056Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fa5435fef6c304211db7e61872224126396daea2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T141010846056Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8ed8e371b89dc085c2792f3c14ce3d4c364e7ad236743c320ec81e56ee72eae +size 5623091 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T141011997558Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T141011997558Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cbd83fee21bedd9908971b2012b874a358919026 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T141011997558Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_peak": 8760513536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:11.997140Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5324131328, + "inactive_split_bytes_peak": 8760513536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:10.966209Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8760513536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T141011997558Z.pickle", + "timestamp_utc": "2026-08-01T14:10:12.102475Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T141011997558Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T141011997558Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d4e703f0793ee8392320d6c8237a1ed6bf2f1a22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T141011997558Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:883b8442fe60c0804771ea79ce1cc341e8ef1b40d1dc973bc32ac23e9c9eb92d +size 5624301 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T141013452094Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T141013452094Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cec103009becfd1f451d67bd9a7fda752141a9e8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T141013452094Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5315742720, + "inactive_split_bytes_peak": 8760513536, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:13.451677Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5322034176, + "inactive_split_bytes_peak": 8760513536, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:12.117007Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5315742720, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 8760513536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T141013452094Z.pickle", + "timestamp_utc": "2026-08-01T14:10:13.559282Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T141013452094Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T141013452094Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..edc5e964930b49f8fddf81a647587e0263ff2e1d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T141013452094Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a04cb9b5bb71fb0364b4a0a6fef0867198aaa5c10c1127e525db53455d15d37f +size 5622830 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T141014609774Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T141014609774Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0bf4654b0a7e137f25c060e0e833d1a334298461 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T141014609774Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 8760513536, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:14.609361Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5315742720, + "inactive_split_bytes_peak": 8760513536, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:13.573844Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 8760513536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T141014609774Z.pickle", + "timestamp_utc": "2026-08-01T14:10:14.712481Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T141014609774Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T141014609774Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e3fabc1565ded1f49b55ba8b5f160a6c9620faa8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T141014609774Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:985ca9ab41b9f79a562ac577a7259245417e40e4989500b1395746726d1f3d80 +size 5624146 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T141015757481Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T141015757481Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a9f3f3f8181c95e18ea14aebad74869700d4f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T141015757481Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 8760513536, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:15.757074Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 8760513536, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:14.727294Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 8760513536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T141015757481Z.pickle", + "timestamp_utc": "2026-08-01T14:10:15.865740Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T141015757481Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T141015757481Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..97eecd8befd20b3dc44be28377542d1e7db7e988 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T141015757481Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94017d49d93f3b9f3c479c6af600c10b4c32959a2e84b5caa8bea895a7b69a0b +size 5624214 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T141016912927Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T141016912927Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3afa1e12b976cfc417d1df25e7c53544d9e19255 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T141016912927Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5317839872, + "inactive_split_bytes_peak": 8760513536, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:16.912499Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 8760513536, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:15.882797Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5317839872, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 8760513536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T141016912927Z.pickle", + "timestamp_utc": "2026-08-01T14:10:17.015902Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T141016912927Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T141016912927Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d93489d37e39824b304417bf910bb3ab887913f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T141016912927Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff299e5401996617405ba821b0965a4821c3140d50b362c299f60330903f8c33 +size 5624128 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T141018065663Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T141018065663Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d7b0c63d45a15e58a2d72557d3c0b808ac98081 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T141018065663Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 8760513536, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:18.065260Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5317839872, + "inactive_split_bytes_peak": 8760513536, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:17.030210Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 8760513536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T141018065663Z.pickle", + "timestamp_utc": "2026-08-01T14:10:18.168828Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T141018065663Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T141018065663Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..857f093b384bfd9999889374fccd141c656ab4c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T141018065663Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58dd1b1f32ee49aa74d60b51bf581118876a7865bddce4e9fec0f2c3cdcd95ba +size 5624184 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T141115593719Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T141115593719Z.json new file mode 100644 index 0000000000000000000000000000000000000000..89ed991147012f00c95d0e0ee56ca72042155fa7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T141115593719Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110728266752, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5047618560, + "inactive_split_bytes_peak": 8760513536, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:11:15.593246Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110728266752, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 110728266752, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 110728266752, + "device": 6, + "inactive_split_bytes_current": 5319937024, + "inactive_split_bytes_peak": 8760513536, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 113476894720, + "reserved_bytes_peak": 113476894720, + "segment_count_current": 566, + "segment_count_peak": 566, + "timestamp_utc": "2026-08-01T14:10:18.183183Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5047618560, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 8760513536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 113476894720, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113476894720, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 566, + "segment_count_current_delta": 0, + "segment_count_peak": 566, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T141115593719Z.pickle", + "timestamp_utc": "2026-08-01T14:11:15.792646Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T141115593719Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T141115593719Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..314762544145bf6bbf7406afdd39a51f850e7010 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T141115593719Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f1b85d86dd68432edc2b8c0f656398a5dab51790d2ef4603d0a783c0a8aa30b +size 9552246 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T141008516606Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T141008516606Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b851a56e29bf5a49560a9fa7b14bec135020c179 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T141008516606Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 31622094336, + "active_bytes_peak": 87096555008, + "active_bytes_peak_delta": 51630218752, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 87096555008, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 87096555008, + "device": 7, + "inactive_split_bytes_current": 5544331776, + "inactive_split_bytes_peak": 9228178432, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 96569655296, + "reserved_bytes_peak": 96569655296, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T14:10:08.516123Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 31622094336, + "allocated_bytes_peak": 87096555008, + "allocated_bytes_peak_delta": 51630218752, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:09:01.301226Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5544331776, + "inactive_split_bytes_current_delta": 5411512832, + "inactive_split_bytes_peak": 9228178432, + "inactive_split_bytes_peak_delta": 9067792384, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 96569655296, + "reserved_bytes_current_delta": 57325649920, + "reserved_bytes_peak": 96569655296, + "reserved_bytes_peak_delta": 57325649920, + "segment_count_current": 498, + "segment_count_current_delta": 427, + "segment_count_peak": 498, + "segment_count_peak_delta": 427, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T141008516606Z.pickle", + "timestamp_utc": "2026-08-01T14:10:08.627164Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T141008516606Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T141008516606Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..778a0647125fe6d29b0ce8aa548a013a3c108733 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T141008516606Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36cd0bde6ce46f360b5cd5b65a2a18768693c119cde163de35c5df3ea8c5018c +size 5397133 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T141009707968Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T141009707968Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d41db487c8ca9bf6ee5c8eeabde2d712fb3de974 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T141009707968Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110727493120, + "active_bytes_peak_delta": 23630938112, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110727493120, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110727493120, + "device": 7, + "inactive_split_bytes_current": 5328325120, + "inactive_split_bytes_peak": 9228178432, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:09.707559Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110727493120, + "allocated_bytes_peak_delta": 23630938112, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 87096555008, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 87096555008, + "device": 7, + "inactive_split_bytes_current": 5544331776, + "inactive_split_bytes_peak": 9228178432, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 96569655296, + "reserved_bytes_peak": 96569655296, + "segment_count_current": 498, + "segment_count_peak": 498, + "timestamp_utc": "2026-08-01T14:10:08.644639Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5328325120, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 9228178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 17377001472, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 17377001472, + "segment_count_current": 568, + "segment_count_current_delta": 70, + "segment_count_peak": 568, + "segment_count_peak_delta": 70, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T141009707968Z.pickle", + "timestamp_utc": "2026-08-01T14:10:09.820285Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T141009707968Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T141009707968Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d3ca7813e150ed378cbabcdff46524a2bf47bb3e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T141009707968Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68d23c0cf0913daaa11fa20d00767dd1d01f9930c18478d48af24403a2f124c8 +size 5628186 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T141010846053Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T141010846053Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5618ca991319f3bd620232c7ca90c5cd78fa3bcf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T141010846053Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728267264, + "active_bytes_peak_delta": 774144, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5326227968, + "inactive_split_bytes_peak": 9228178432, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:10.845654Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728267264, + "allocated_bytes_peak_delta": 774144, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110727493120, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110727493120, + "device": 7, + "inactive_split_bytes_current": 5328325120, + "inactive_split_bytes_peak": 9228178432, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:09.834184Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5326227968, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 9228178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T141010846053Z.pickle", + "timestamp_utc": "2026-08-01T14:10:10.949008Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T141010846053Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T141010846053Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6d09e6e9fe7c6d0e5524db2947a3e4feedeb0fcc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T141010846053Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a3af99959bb4abe0e8add33c758bd93a493dcb7f7e4c0d9489a76e73b5c2b43 +size 5625110 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T141011997625Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T141011997625Z.json new file mode 100644 index 0000000000000000000000000000000000000000..756e9bad0ad51393f495f95541d89d6bfc08fd8d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T141011997625Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728267264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5559011840, + "inactive_split_bytes_peak": 9228178432, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:11.997234Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728267264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5326227968, + "inactive_split_bytes_peak": 9228178432, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:10.962281Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5559011840, + "inactive_split_bytes_current_delta": 232783872, + "inactive_split_bytes_peak": 9228178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T141011997625Z.pickle", + "timestamp_utc": "2026-08-01T14:10:12.099312Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T141011997625Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T141011997625Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80ad6e335a1ef73b35e961e9595b6c73149352c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T141011997625Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2232b015a50663c55206b9b77eda437e510990da687c281ad9874576a17a1c90 +size 5626416 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T141013452141Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T141013452141Z.json new file mode 100644 index 0000000000000000000000000000000000000000..058fd7c38125f6964891ea77f9694fcb512ae909 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T141013452141Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728267264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5317839360, + "inactive_split_bytes_peak": 9228178432, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:13.451734Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728267264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5559011840, + "inactive_split_bytes_peak": 9228178432, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:12.112700Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5317839360, + "inactive_split_bytes_current_delta": -241172480, + "inactive_split_bytes_peak": 9228178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T141013452141Z.pickle", + "timestamp_utc": "2026-08-01T14:10:13.554774Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T141013452141Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T141013452141Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9415b15ee2c22cbdac5087b5d2005e181aea3e75 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T141013452141Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59db1c44e68853c362a048064d807590b00d83bbc1e94b5498ae5f1df2819a38 +size 5624854 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T141014609851Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T141014609851Z.json new file mode 100644 index 0000000000000000000000000000000000000000..74dab63eca46a119a76e414bc148ff47baff36e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T141014609851Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728267264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5322033664, + "inactive_split_bytes_peak": 9228178432, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:14.609437Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728267264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5317839360, + "inactive_split_bytes_peak": 9228178432, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:13.567110Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5322033664, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 9228178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T141014609851Z.pickle", + "timestamp_utc": "2026-08-01T14:10:14.708968Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T141014609851Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T141014609851Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..afcd78b90466ffeea63cf317c3d2f7e15276bc79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T141014609851Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f800990bf405deb584afee95fab7d0601b1da05f1a8383e51a83851cd90300f0 +size 5626063 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T141015757575Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T141015757575Z.json new file mode 100644 index 0000000000000000000000000000000000000000..088219efe639f9e2882720f3b1da92ed0a6ec3c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T141015757575Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728267264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5556914688, + "inactive_split_bytes_peak": 9228178432, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:15.757176Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728267264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5322033664, + "inactive_split_bytes_peak": 9228178432, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:14.721097Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5556914688, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 9228178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T141015757575Z.pickle", + "timestamp_utc": "2026-08-01T14:10:15.855192Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T141015757575Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T141015757575Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2c2b28917ba2b7b07c7272c452d996f7cecca257 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T141015757575Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b6ab73258b0bc8a790d5f88850855109c155b19fcf12098ccf1c799af83265d +size 5626124 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T141016912973Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T141016912973Z.json new file mode 100644 index 0000000000000000000000000000000000000000..81fb97d31a92ba939af140fd569f4097ce99f1fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T141016912973Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728267264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5319936512, + "inactive_split_bytes_peak": 9228178432, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:16.912551Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728267264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5556914688, + "inactive_split_bytes_peak": 9228178432, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:15.867325Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5319936512, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 9228178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T141016912973Z.pickle", + "timestamp_utc": "2026-08-01T14:10:17.012713Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T141016912973Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T141016912973Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a88a4da522e5bcf9488bbc2c94563a7c8a3c5cad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T141016912973Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acb39146cc8e0d9cc26934ca1ca48e05d00d8277c1e2f2316ebc4ed6f4824e8e +size 5626051 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T141018065751Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T141018065751Z.json new file mode 100644 index 0000000000000000000000000000000000000000..97dd534763c8d173d1ab2d994ae651d8f5d8f5af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T141018065751Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110728267264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5322033664, + "inactive_split_bytes_peak": 9228178432, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:18.065350Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110728267264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5319936512, + "inactive_split_bytes_peak": 9228178432, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:17.024859Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5322033664, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 9228178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T141018065751Z.pickle", + "timestamp_utc": "2026-08-01T14:10:18.161678Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T141018065751Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T141018065751Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..85de7eab2e27224835f7e7d93c080551432c3d45 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T141018065751Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:166f747449246fe25fb325f33f066645af0dfb3b956b9137af77fe64c08a2ce6 +size 5626164 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T141115593692Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T141115593692Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a14000df53b39df1b55876caebd78092d8171f45 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T141115593692Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906624, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 110728267264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906624, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66802906624, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5049715200, + "inactive_split_bytes_peak": 9228178432, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:11:15.593234Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906624, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 110728267264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 110728267264, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 110728267264, + "device": 7, + "inactive_split_bytes_current": 5322033664, + "inactive_split_bytes_peak": 9228178432, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 113946656768, + "reserved_bytes_peak": 113946656768, + "segment_count_current": 568, + "segment_count_peak": 568, + "timestamp_utc": "2026-08-01T14:10:18.173311Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5049715200, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 9228178432, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 113946656768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 113946656768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 568, + "segment_count_current_delta": 0, + "segment_count_peak": 568, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T141115593692Z.pickle", + "timestamp_utc": "2026-08-01T14:11:15.786175Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T141115593692Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T141115593692Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1700261f90af5e8e3107957e6f6fa9ce9cca2136 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T141115593692Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9163c17806672540b51fac969b6466b9ee1946f76935b17ee1ce00057d256e05 +size 9554176 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..6cefe4ffdca77b6e1e8c0219884ddefff3e3d0da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 4096 +capacity: 1.00 +expert_dispatch_overlap: on +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6039 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 226 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T141400754129Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T141400754129Z.json new file mode 100644 index 0000000000000000000000000000000000000000..72bf5c5a48971b9fd6e6616829f5f3e229778e01 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T141400754129Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440716288, + "active_bytes_current_delta": 37209261056, + "active_bytes_peak": 87350381568, + "active_bytes_peak_delta": 51884045312, + "after": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 87350381568, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 86998060032, + "device": 0, + "inactive_split_bytes_current": 5917271040, + "inactive_split_bytes_peak": 10794548224, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 97611939840, + "reserved_bytes_peak": 97611939840, + "segment_count_current": 452, + "segment_count_peak": 452, + "timestamp_utc": "2026-08-01T14:14:00.753553Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440716288, + "allocated_bytes_current_delta": 37209261056, + "allocated_bytes_peak": 86998060032, + "allocated_bytes_peak_delta": 51531723776, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T14:12:52.067270Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5917271040, + "inactive_split_bytes_current_delta": 5782354944, + "inactive_split_bytes_peak": 10794548224, + "inactive_split_bytes_peak_delta": 10633916416, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 97611939840, + "reserved_bytes_current_delta": 58365837312, + "reserved_bytes_peak": 97611939840, + "reserved_bytes_peak_delta": 58365837312, + "segment_count_current": 452, + "segment_count_current_delta": 380, + "segment_count_peak": 452, + "segment_count_peak_delta": 380, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T141400754129Z.pickle", + "timestamp_utc": "2026-08-01T14:14:00.869591Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T141400754129Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T141400754129Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a6f6be4cd423814c4619d988dd25eb45b3f6ee5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T141400754129Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bcc8f5da04548e3e673b8bf0d8ea9afc75e98a0440c7d8e873dbf7920acf166 +size 5965612 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T141402053057Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T141402053057Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1edcfd7543389ba0501cc7f0d457c820d7efc228 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T141402053057Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440716288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110982742528, + "active_bytes_peak_delta": 23632360960, + "after": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 110982742528, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110630420992, + "device": 0, + "inactive_split_bytes_current": 7504815104, + "inactive_split_bytes_peak": 11293031424, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115869745152, + "reserved_bytes_peak": 115869745152, + "segment_count_current": 528, + "segment_count_peak": 528, + "timestamp_utc": "2026-08-01T14:14:02.052671Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440716288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110630420992, + "allocated_bytes_peak_delta": 23632360960, + "before": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 87350381568, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 86998060032, + "device": 0, + "inactive_split_bytes_current": 5917271040, + "inactive_split_bytes_peak": 10794548224, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 97611939840, + "reserved_bytes_peak": 97611939840, + "segment_count_current": 452, + "segment_count_peak": 452, + "timestamp_utc": "2026-08-01T14:14:00.885385Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7504815104, + "inactive_split_bytes_current_delta": 1587544064, + "inactive_split_bytes_peak": 11293031424, + "inactive_split_bytes_peak_delta": 498483200, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115869745152, + "reserved_bytes_current_delta": 18257805312, + "reserved_bytes_peak": 115869745152, + "reserved_bytes_peak_delta": 18257805312, + "segment_count_current": 528, + "segment_count_current_delta": 76, + "segment_count_peak": 528, + "segment_count_peak_delta": 76, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T141402053057Z.pickle", + "timestamp_utc": "2026-08-01T14:14:02.167745Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T141402053057Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T141402053057Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f5e629340aed6eee00b26ddc7d8f6741caf3e61 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T141402053057Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dbaf4b5c7e82e16c62b1cab2778c0eec9d57e58cfb45e3bb1b5d384180c50c9 +size 6663200 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T141403334272Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T141403334272Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76cb3c3de1300c285db0b6514badee4719c25058 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T141403334272Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440716288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111328980480, + "active_bytes_peak_delta": 346237952, + "after": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111328980480, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 8092017664, + "inactive_split_bytes_peak": 11293031424, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115869745152, + "reserved_bytes_peak": 115869745152, + "segment_count_current": 528, + "segment_count_peak": 528, + "timestamp_utc": "2026-08-01T14:14:03.333885Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440716288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110631619584, + "allocated_bytes_peak_delta": 1198592, + "before": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 110982742528, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110630420992, + "device": 0, + "inactive_split_bytes_current": 7504815104, + "inactive_split_bytes_peak": 11293031424, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115869745152, + "reserved_bytes_peak": 115869745152, + "segment_count_current": 528, + "segment_count_peak": 528, + "timestamp_utc": "2026-08-01T14:14:02.180696Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8092017664, + "inactive_split_bytes_current_delta": 587202560, + "inactive_split_bytes_peak": 11293031424, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115869745152, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115869745152, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 528, + "segment_count_current_delta": 0, + "segment_count_peak": 528, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T141403334272Z.pickle", + "timestamp_utc": "2026-08-01T14:14:03.434158Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T141403334272Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T141403334272Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c93ace7649425d473317cc8b97be15499ca4d0d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T141403334272Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:506ee82e07c9ebbeb29cfdf68731a1406c3af94f9e0c14ad336f33d988e14a57 +size 6660256 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T141404630629Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T141404630629Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f424a73e3e33758a0f502ba3c87f9ba681b4866 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T141404630629Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440716288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111328980480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111328980480, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 9031541760, + "inactive_split_bytes_peak": 11345040384, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117163687936, + "reserved_bytes_peak": 117163687936, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:04.630230Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440716288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110631619584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111328980480, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 8092017664, + "inactive_split_bytes_peak": 11293031424, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115869745152, + "reserved_bytes_peak": 115869745152, + "segment_count_current": 528, + "segment_count_peak": 528, + "timestamp_utc": "2026-08-01T14:14:03.445965Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9031541760, + "inactive_split_bytes_current_delta": 939524096, + "inactive_split_bytes_peak": 11345040384, + "inactive_split_bytes_peak_delta": 52008960, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117163687936, + "reserved_bytes_current_delta": 1293942784, + "reserved_bytes_peak": 117163687936, + "reserved_bytes_peak_delta": 1293942784, + "segment_count_current": 530, + "segment_count_current_delta": 2, + "segment_count_peak": 530, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T141404630629Z.pickle", + "timestamp_utc": "2026-08-01T14:14:04.739500Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T141404630629Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T141404630629Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..81cf2d787536d2d12b01955fee365cb1930d98dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T141404630629Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96f20e6a875b3efcc46753389162bbf7ed9bf3ec1074f6ab50e659acb8593e19 +size 6659209 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T141406196642Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T141406196642Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1414b3517334a269c41f4184438324e2178d41 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T141406196642Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440716288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111328980480, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111328980480, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 9266422784, + "inactive_split_bytes_peak": 12066190336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_peak": 117987868672, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:06.196200Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440716288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110631619584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111328980480, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 9031541760, + "inactive_split_bytes_peak": 11345040384, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117163687936, + "reserved_bytes_peak": 117163687936, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:04.752176Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9266422784, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 12066190336, + "inactive_split_bytes_peak_delta": 721149952, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117987868672, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 531, + "segment_count_current_delta": 1, + "segment_count_peak": 531, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T141406196642Z.pickle", + "timestamp_utc": "2026-08-01T14:14:06.303678Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T141406196642Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T141406196642Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1b2e87343b5d11a25d7c3f31842082bbf3afd474 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T141406196642Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:687395a41730fe16e5a728bb99dab3b349752eae27c9542555236dc2e7f0f872 +size 6658746 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T141407489965Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T141407489965Z.json new file mode 100644 index 0000000000000000000000000000000000000000..11c55ad347a283f49317c6d742d78ea72ba734ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T141407489965Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440716288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111329195008, + "active_bytes_peak_delta": 214528, + "after": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111329195008, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 8679220224, + "inactive_split_bytes_peak": 12066190336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_peak": 117987868672, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:07.489577Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440716288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110631619584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111328980480, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 9266422784, + "inactive_split_bytes_peak": 12066190336, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_peak": 117987868672, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:06.316389Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8679220224, + "inactive_split_bytes_current_delta": -587202560, + "inactive_split_bytes_peak": 12066190336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117987868672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 531, + "segment_count_current_delta": 0, + "segment_count_peak": 531, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T141407489965Z.pickle", + "timestamp_utc": "2026-08-01T14:14:07.588632Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T141407489965Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T141407489965Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80c462637ae0daa86e3c79ae4da30a50c83be7ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T141407489965Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e72a50b2c1ab8472fc33a231a17d4e2a1bc91098c12d380fc257e7667bb13b41 +size 6658161 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T141408787214Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T141408787214Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6fd52f6fa045f57bb08719dcbdc20d1f497e9cf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T141408787214Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440716288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111332354048, + "active_bytes_peak_delta": 3159040, + "after": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111332354048, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 7974577152, + "inactive_split_bytes_peak": 12066190336, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_peak": 117987868672, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:08.786803Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440716288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110631619584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111329195008, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 8679220224, + "inactive_split_bytes_peak": 12066190336, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_peak": 117987868672, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:07.600415Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7974577152, + "inactive_split_bytes_current_delta": -704643072, + "inactive_split_bytes_peak": 12066190336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117987868672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 531, + "segment_count_current_delta": 0, + "segment_count_peak": 531, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T141408787214Z.pickle", + "timestamp_utc": "2026-08-01T14:14:08.891893Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T141408787214Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T141408787214Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7cb212d8942b34a64741cfeec833a1eb05743bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T141408787214Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba084b11f30ac7fc17eca5c2c0698db25f7c1165a4d439d6e8fd6f4a37d53fed +size 6658050 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T141410231383Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T141410231383Z.json new file mode 100644 index 0000000000000000000000000000000000000000..96970e10718f190375bbb0b56b5ad05b2da36754 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T141410231383Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440716288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111332354048, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111332354048, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 7857136640, + "inactive_split_bytes_peak": 12066190336, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_peak": 117987868672, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:10.230963Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440716288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110631619584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111332354048, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 7974577152, + "inactive_split_bytes_peak": 12066190336, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_peak": 117987868672, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:08.904262Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7857136640, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 12066190336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117987868672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 531, + "segment_count_current_delta": 0, + "segment_count_peak": 531, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T141410231383Z.pickle", + "timestamp_utc": "2026-08-01T14:14:10.336639Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T141410231383Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T141410231383Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb22ed7a19f49fa228e05ae735b479d8730579ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T141410231383Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:778e5fd767ecf045a995ad0f47f6be86f3dcac01d00d6572b335562bedb61db3 +size 6657454 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T141411517540Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T141411517540Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3862cd0c3287cb5b17c66c0cc5cde76287465481 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T141411517540Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440716288, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111332354048, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111332354048, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 8853283840, + "inactive_split_bytes_peak": 12066190336, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_peak": 117987868672, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:11.517137Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440716288, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110631619584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111332354048, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 7857136640, + "inactive_split_bytes_peak": 12066190336, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_peak": 117987868672, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:10.349247Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8853283840, + "inactive_split_bytes_current_delta": 996147200, + "inactive_split_bytes_peak": 12066190336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117987868672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 531, + "segment_count_current_delta": 0, + "segment_count_peak": 531, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T141411517540Z.pickle", + "timestamp_utc": "2026-08-01T14:14:11.616346Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T141411517540Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T141411517540Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f0d33dfc67f402b5494f13c4919e0583289fb70b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T141411517540Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23f8d8a6356e6e3363c5eb321e9cd700d1bb40ed51034640e1f685f161270dd5 +size 6657331 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T141504482941Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T141504482941Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c543cd65cd2ddcaae1a6372f2d48f77ae75a4936 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T141504482941Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166427136, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 111332354048, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166427136, + "active_bytes_peak": 111332354048, + "allocated_bytes_current": 61166427136, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 7678878720, + "inactive_split_bytes_peak": 12066190336, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_peak": 117987868672, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:15:04.482497Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166427136, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110631619584, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440716288, + "active_bytes_peak": 111332354048, + "allocated_bytes_current": 72440716288, + "allocated_bytes_peak": 110631619584, + "device": 0, + "inactive_split_bytes_current": 8853283840, + "inactive_split_bytes_peak": 12066190336, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_peak": 117987868672, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:11.628139Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7678878720, + "inactive_split_bytes_current_delta": -1174405120, + "inactive_split_bytes_peak": 12066190336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117987868672, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117987868672, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 531, + "segment_count_current_delta": 0, + "segment_count_peak": 531, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T141504482941Z.pickle", + "timestamp_utc": "2026-08-01T14:15:04.682839Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T141504482941Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T141504482941Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7bb105eaa360886db786ab3954ea15b779c0d4b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T141504482941Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2e924c214ca6eb6da5411cd84bc8a9c05fbde21e36bfa4bc96c398d3af3b4a6 +size 10558368 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T141400754219Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T141400754219Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5e36a470f8f346c9d32b931f59b9367ea4a09752 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T141400754219Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 87350904832, + "active_bytes_peak_delta": 51884568576, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87350904832, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 86998583296, + "device": 1, + "inactive_split_bytes_current": 5989239808, + "inactive_split_bytes_peak": 10503272448, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 97609842688, + "reserved_bytes_peak": 97609842688, + "segment_count_current": 452, + "segment_count_peak": 452, + "timestamp_utc": "2026-08-01T14:14:00.753583Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 86998583296, + "allocated_bytes_peak_delta": 51532247040, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:12:52.067085Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5989239808, + "inactive_split_bytes_current_delta": 5856420864, + "inactive_split_bytes_peak": 10503272448, + "inactive_split_bytes_peak_delta": 10342886400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 97609842688, + "reserved_bytes_current_delta": 58365837312, + "reserved_bytes_peak": 97609842688, + "reserved_bytes_peak_delta": 58365837312, + "segment_count_current": 452, + "segment_count_current_delta": 381, + "segment_count_peak": 452, + "segment_count_peak_delta": 381, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T141400754219Z.pickle", + "timestamp_utc": "2026-08-01T14:14:00.877222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T141400754219Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T141400754219Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8b0d7020954a842af48824ee4240d42b0993a518 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T141400754219Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e3a7e31c1781e6fa9f118647e2c1e1c41503db3d647370d962d851406cf1eed +size 5955539 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T141402053101Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T141402053101Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9d02be6a72e9e23baf767283a84ea84719b1beb3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T141402053101Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110980370944, + "active_bytes_peak_delta": 23629466112, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110980370944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110628049408, + "device": 1, + "inactive_split_bytes_current": 7811664896, + "inactive_split_bytes_peak": 11416431616, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:02.052689Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110628049408, + "allocated_bytes_peak_delta": 23629466112, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87350904832, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 86998583296, + "device": 1, + "inactive_split_bytes_current": 5989239808, + "inactive_split_bytes_peak": 10503272448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 97609842688, + "reserved_bytes_peak": 97609842688, + "segment_count_current": 452, + "segment_count_peak": 452, + "timestamp_utc": "2026-08-01T14:14:00.896126Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7811664896, + "inactive_split_bytes_current_delta": 1822425088, + "inactive_split_bytes_peak": 11416431616, + "inactive_split_bytes_peak_delta": 913159168, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116576485376, + "reserved_bytes_current_delta": 18966642688, + "reserved_bytes_peak": 116576485376, + "reserved_bytes_peak_delta": 18966642688, + "segment_count_current": 530, + "segment_count_current_delta": 78, + "segment_count_peak": 530, + "segment_count_peak_delta": 78, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T141402053101Z.pickle", + "timestamp_utc": "2026-08-01T14:14:02.171751Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T141402053101Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T141402053101Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..245a74b675d6ca9070122fee36870d4462098aec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T141402053101Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27c6c5b20db9b2dfc3a00f95b6581690cefec8acd7e3da32071641b6bb599870 +size 6119881 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T141403334277Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T141403334277Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87a0cdc5d5838f34858ef6b1d8705f6f8831594c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T141403334277Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111329036288, + "active_bytes_peak_delta": 348665344, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111329036288, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110628049408, + "device": 1, + "inactive_split_bytes_current": 7868288000, + "inactive_split_bytes_peak": 11416431616, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:03.333886Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110628049408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110980370944, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110628049408, + "device": 1, + "inactive_split_bytes_current": 7811664896, + "inactive_split_bytes_peak": 11416431616, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:02.188050Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7868288000, + "inactive_split_bytes_current_delta": 56623104, + "inactive_split_bytes_peak": 11416431616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116576485376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116576485376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T141403334277Z.pickle", + "timestamp_utc": "2026-08-01T14:14:03.446088Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T141403334277Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T141403334277Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fa4e8b788b0bf8023fc9ab4ef577fd6e906e0f6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T141403334277Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9b0eb226e52ec6d702bb492a9ef87742875790d354eee7e1be384c8dd3786d4 +size 6639148 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T141404630611Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T141404630611Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fafd3b38de071a3d4d0ac841134a0cec67fb4543 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T141404630611Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111337865216, + "active_bytes_peak_delta": 8828928, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 7046204416, + "inactive_split_bytes_peak": 11416431616, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:04.630211Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110633222144, + "allocated_bytes_peak_delta": 5172736, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111329036288, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110628049408, + "device": 1, + "inactive_split_bytes_current": 7868288000, + "inactive_split_bytes_peak": 11416431616, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:03.461914Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7046204416, + "inactive_split_bytes_current_delta": -822083584, + "inactive_split_bytes_peak": 11416431616, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116576485376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116576485376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T141404630611Z.pickle", + "timestamp_utc": "2026-08-01T14:14:04.745519Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T141404630611Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T141404630611Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4de8729cc63bc6e270febc5a8f0387796e13e6d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T141404630611Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8373ab56f1f4d6796a218065c57d8ce3db79bb394be46ee1fb713dde78e4a574 +size 6638553 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T141406196597Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T141406196597Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f44a28ca73d258dc4e73877444057ab1cd19089 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T141406196597Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111337865216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 7750847488, + "inactive_split_bytes_peak": 11603767296, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:06.196195Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110633222144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 7046204416, + "inactive_split_bytes_peak": 11416431616, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:04.761212Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7750847488, + "inactive_split_bytes_current_delta": 704643072, + "inactive_split_bytes_peak": 11603767296, + "inactive_split_bytes_peak_delta": 187335680, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116576485376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116576485376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T141406196597Z.pickle", + "timestamp_utc": "2026-08-01T14:14:06.307462Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T141406196597Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T141406196597Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6239463deb168cbf7641d7d7cd64fa7116ebb788 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T141406196597Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb507432ec4a29beba1050351340710e25f24a48b2004ff2034e03028a433f11 +size 6638012 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T141407489989Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T141407489989Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0aeb30e90232b4dd337af52c4b91adfec505d615 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T141407489989Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111337865216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 8516307968, + "inactive_split_bytes_peak": 11603767296, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:07.489561Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110633222144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 7750847488, + "inactive_split_bytes_peak": 11603767296, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:06.322940Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8516307968, + "inactive_split_bytes_current_delta": 765460480, + "inactive_split_bytes_peak": 11603767296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116576485376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116576485376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T141407489989Z.pickle", + "timestamp_utc": "2026-08-01T14:14:07.599238Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T141407489989Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T141407489989Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c233c7040668f2476e0cdddcf514a82683439f75 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T141407489989Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:462da97643adf3344612ba9e44e57944bac5a0537b065e61065d3041df46886c +size 6637956 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T141408787216Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T141408787216Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fe752583504c46bc4ddf28e1a7aeed34e0c19d19 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T141408787216Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111337865216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 8572931072, + "inactive_split_bytes_peak": 11603767296, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:08.786797Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110633222144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 8516307968, + "inactive_split_bytes_peak": 11603767296, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:07.614659Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8572931072, + "inactive_split_bytes_current_delta": 56623104, + "inactive_split_bytes_peak": 11603767296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116576485376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116576485376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T141408787216Z.pickle", + "timestamp_utc": "2026-08-01T14:14:08.895418Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T141408787216Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T141408787216Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..418fed3f32c0251d30bf697f516fb1d5fca6921b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T141408787216Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f4c5c0011a66e87176ab31cf6f954660862b3632b7352d50767b20dd4719d8a +size 6637495 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T141410231384Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T141410231384Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2cd9d67ff2141f52b88ffc6091b61ffae4d358a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T141410231384Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111337865216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 7868288000, + "inactive_split_bytes_peak": 11603767296, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:10.230949Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110633222144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 8572931072, + "inactive_split_bytes_peak": 11603767296, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:08.910748Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7868288000, + "inactive_split_bytes_current_delta": -704643072, + "inactive_split_bytes_peak": 11603767296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116576485376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116576485376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T141410231384Z.pickle", + "timestamp_utc": "2026-08-01T14:14:10.341774Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T141410231384Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T141410231384Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..077a927a267fe68c4fb61656b6189493f98c253d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T141410231384Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d49c0882caed7396bb29236d72e0d32a51389ac33b16de7dba706dcb0cafaef +size 6637367 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T141411517531Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T141411517531Z.json new file mode 100644 index 0000000000000000000000000000000000000000..82e8bfb49ca122c14b2a9bb00f409665d502583a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T141411517531Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111337865216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 8163986432, + "inactive_split_bytes_peak": 11603767296, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:11.517127Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110633222144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 7868288000, + "inactive_split_bytes_peak": 11603767296, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:10.357199Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8163986432, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 11603767296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116576485376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116576485376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T141411517531Z.pickle", + "timestamp_utc": "2026-08-01T14:14:11.624857Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T141411517531Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T141411517531Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4aadfe12e018bbbb60bb253029d61bf26a2769da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T141411517531Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bfbfb94e79f38362ced7d254fc9d5acb7ffe485146549598722242f80dea51c +size 6637072 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T141504483076Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T141504483076Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b5a82dc8960aa6464b4a3209885f5afcf9fe7e0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T141504483076Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 111337865216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 7455149056, + "inactive_split_bytes_peak": 11603767296, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:15:04.482509Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110633222144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337865216, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110633222144, + "device": 1, + "inactive_split_bytes_current": 8163986432, + "inactive_split_bytes_peak": 11603767296, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116576485376, + "reserved_bytes_peak": 116576485376, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:11.640148Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7455149056, + "inactive_split_bytes_current_delta": -708837376, + "inactive_split_bytes_peak": 11603767296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116576485376, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116576485376, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T141504483076Z.pickle", + "timestamp_utc": "2026-08-01T14:15:04.697540Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T141504483076Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T141504483076Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..69dad20110730ade5bd70dcde4e8b9baf7001a00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T141504483076Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cee40c808a9a1a249a7de635c2549eec4a1f33dddfc2e4ef81a39620d2366ff +size 9977884 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T141400754069Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T141400754069Z.json new file mode 100644 index 0000000000000000000000000000000000000000..98d2cf937b40b1565dde7fd4e38b007b8183a31a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T141400754069Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 87356409856, + "active_bytes_peak_delta": 51890073600, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87356409856, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 87004088320, + "device": 2, + "inactive_split_bytes_current": 5517380608, + "inactive_split_bytes_peak": 9606480896, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 97196703744, + "reserved_bytes_peak": 97196703744, + "segment_count_current": 451, + "segment_count_peak": 451, + "timestamp_utc": "2026-08-01T14:14:00.753482Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 87004088320, + "allocated_bytes_peak_delta": 51537752064, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:12:52.066878Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5517380608, + "inactive_split_bytes_current_delta": 5384561664, + "inactive_split_bytes_peak": 9606480896, + "inactive_split_bytes_peak_delta": 9446094848, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 97196703744, + "reserved_bytes_current_delta": 57952698368, + "reserved_bytes_peak": 97196703744, + "reserved_bytes_peak_delta": 57952698368, + "segment_count_current": 451, + "segment_count_current_delta": 380, + "segment_count_peak": 451, + "segment_count_peak_delta": 380, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T141400754069Z.pickle", + "timestamp_utc": "2026-08-01T14:14:00.870211Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T141400754069Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T141400754069Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e2e818b2d749d10fc593472236f1867762d4364 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T141400754069Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c87c6a2bcaecd16675bfed132ff2db330500c8976e3dc17640356ea0920676c +size 5955335 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T141402053151Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T141402053151Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3b36b96d8deeb2772f9a168218d8440cda419ad0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T141402053151Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110986599424, + "active_bytes_peak_delta": 23630189568, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110986599424, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 8218512384, + "inactive_split_bytes_peak": 11351000064, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116280786944, + "reserved_bytes_peak": 116280786944, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:02.052711Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110634277888, + "allocated_bytes_peak_delta": 23630189568, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87356409856, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 87004088320, + "device": 2, + "inactive_split_bytes_current": 5517380608, + "inactive_split_bytes_peak": 9606480896, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 97196703744, + "reserved_bytes_peak": 97196703744, + "segment_count_current": 451, + "segment_count_peak": 451, + "timestamp_utc": "2026-08-01T14:14:00.887215Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8218512384, + "inactive_split_bytes_current_delta": 2701131776, + "inactive_split_bytes_peak": 11351000064, + "inactive_split_bytes_peak_delta": 1744519168, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116280786944, + "reserved_bytes_current_delta": 19084083200, + "reserved_bytes_peak": 116280786944, + "reserved_bytes_peak_delta": 19084083200, + "segment_count_current": 530, + "segment_count_current_delta": 79, + "segment_count_peak": 530, + "segment_count_peak_delta": 79, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T141402053151Z.pickle", + "timestamp_utc": "2026-08-01T14:14:02.168335Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T141402053151Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T141402053151Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..48e76c15967f8a05d0fbf96fe4dde6a9ac38df72 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T141402053151Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:511cf25e2646728a0e2edd2106a40621298befc0eee0adc014acf42eeeb808a2 +size 6120638 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T141403334323Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T141403334323Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f525d146d46d2399233a0af9afde052dd938fe13 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T141403334323Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111336955904, + "active_bytes_peak_delta": 350356480, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 8453393408, + "inactive_split_bytes_peak": 12359357440, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117104967680, + "reserved_bytes_peak": 117104967680, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:03.333916Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110634277888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110986599424, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 8218512384, + "inactive_split_bytes_peak": 11351000064, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116280786944, + "reserved_bytes_peak": 116280786944, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:02.182500Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8453393408, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 12359357440, + "inactive_split_bytes_peak_delta": 1008357376, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117104967680, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117104967680, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 531, + "segment_count_current_delta": 1, + "segment_count_peak": 531, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T141403334323Z.pickle", + "timestamp_utc": "2026-08-01T14:14:03.440897Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T141403334323Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T141403334323Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a8771881a89df605d8eae5676d09bf47e35274e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T141403334323Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14316f63663c2a2c19abf3ee13e295749cb21ecf750ae576bebdc8a75823f9cc +size 6639960 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T141404630724Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T141404630724Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a2340cfa9d0ced3c4e5c15885c5270500cc0f961 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T141404630724Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111336955904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 8688274432, + "inactive_split_bytes_peak": 12359357440, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:04.630304Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110634277888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 8453393408, + "inactive_split_bytes_peak": 12359357440, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117104967680, + "reserved_bytes_peak": 117104967680, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:03.454292Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8688274432, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 12359357440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117929148416, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117929148416, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 532, + "segment_count_current_delta": 1, + "segment_count_peak": 532, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T141404630724Z.pickle", + "timestamp_utc": "2026-08-01T14:14:04.741529Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T141404630724Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T141404630724Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8b7e95b66ea38797969b5d1432ff3d037665473b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T141404630724Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb9e172f96c379abe6f0dd045c430b9f4e0cb53f7d4f6d9366b7f5c0aa1b3c4a +size 6115745 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T141406196633Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T141406196633Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f1b28bf760e737ed378a690b45f0eed118ece793 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T141406196633Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111336955904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 7396428800, + "inactive_split_bytes_peak": 12359357440, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:06.196223Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110634277888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 8688274432, + "inactive_split_bytes_peak": 12359357440, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:04.755461Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7396428800, + "inactive_split_bytes_current_delta": -1291845632, + "inactive_split_bytes_peak": 12359357440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117929148416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117929148416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T141406196633Z.pickle", + "timestamp_utc": "2026-08-01T14:14:06.307257Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T141406196633Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T141406196633Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..46fdbef67851b69ee1af11dc6b4a1b01c74adde0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T141406196633Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0883ce9dfc0bbfdee03ae9ccc15267cc7e02de619935cf363c8be11a379b041c +size 6637794 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T141407489992Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T141407489992Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ec9c0a78c226251ab909b15915158b9911ba0d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T141407489992Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111336955904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 8453393408, + "inactive_split_bytes_peak": 12359357440, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:07.489591Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110634277888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 7396428800, + "inactive_split_bytes_peak": 12359357440, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:06.321056Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8453393408, + "inactive_split_bytes_current_delta": 1056964608, + "inactive_split_bytes_peak": 12359357440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117929148416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117929148416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T141407489992Z.pickle", + "timestamp_utc": "2026-08-01T14:14:07.600139Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T141407489992Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T141407489992Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..95771c9482be1ae34d6e5588646562e22638fe4e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T141407489992Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2502ba62ebb25b1a61c7f203edf58ad50590ac2733cafb8b14a2622ec2878dff +size 6637381 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T141408787235Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T141408787235Z.json new file mode 100644 index 0000000000000000000000000000000000000000..57811c570a0a33454018dcadfceb3fd5246df689 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T141408787235Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111336955904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 8040254464, + "inactive_split_bytes_peak": 12359357440, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:08.786828Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110634277888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 8453393408, + "inactive_split_bytes_peak": 12359357440, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:07.616013Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8040254464, + "inactive_split_bytes_current_delta": -413138944, + "inactive_split_bytes_peak": 12359357440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117929148416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117929148416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T141408787235Z.pickle", + "timestamp_utc": "2026-08-01T14:14:08.891501Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T141408787235Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T141408787235Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3e3f0da190299f22407b411e924df708748602fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T141408787235Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5a91a89ac21a6477e2e46cc3884753791a39af65c344883155fb93d092cfb97 +size 6637059 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T141410231387Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T141410231387Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6f9607ef82046559dd32358bec6c52480485d35a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T141410231387Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111336955904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 7631309824, + "inactive_split_bytes_peak": 12359357440, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:10.230969Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110634277888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 8040254464, + "inactive_split_bytes_peak": 12359357440, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:08.904825Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7631309824, + "inactive_split_bytes_current_delta": -408944640, + "inactive_split_bytes_peak": 12359357440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117929148416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117929148416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T141410231387Z.pickle", + "timestamp_utc": "2026-08-01T14:14:10.339873Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T141410231387Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T141410231387Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11fa21902a742aab6bdb2958ac91cc5a36569c79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T141410231387Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b4cd594aeeb2ce1fbed9d6798713c006a9b23ec6dc75703aa25e9f05ec11d03 +size 6637070 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T141411517557Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T141411517557Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31b0037a0e5712252e2626456b5b9c8b24bc979b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T141411517557Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 111336955904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 8278281216, + "inactive_split_bytes_peak": 12359357440, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:11.517153Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 110634277888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 7631309824, + "inactive_split_bytes_peak": 12359357440, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:10.353257Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8278281216, + "inactive_split_bytes_current_delta": 646971392, + "inactive_split_bytes_peak": 12359357440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117929148416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117929148416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T141411517557Z.pickle", + "timestamp_utc": "2026-08-01T14:14:11.621318Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T141411517557Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T141411517557Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dc8d052cee20a9b77e864a801f0a447ccdb71514 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T141411517557Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a21c90e138a8322b5fb058a1df034f43f8cc67c0abcec654297c1b8249f6e126 +size 6637203 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T141504482999Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T141504482999Z.json new file mode 100644 index 0000000000000000000000000000000000000000..866be56e84f2c54e8b28f585380c77eb5b47d7d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T141504482999Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11275337728, + "active_bytes_peak": 111336955904, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 7218170880, + "inactive_split_bytes_peak": 12359357440, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:15:04.482539Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11275337728, + "allocated_bytes_peak": 110634277888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 111336955904, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 110634277888, + "device": 2, + "inactive_split_bytes_current": 8278281216, + "inactive_split_bytes_peak": 12359357440, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:11.634629Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7218170880, + "inactive_split_bytes_current_delta": -1060110336, + "inactive_split_bytes_peak": 12359357440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117929148416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117929148416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T141504482999Z.pickle", + "timestamp_utc": "2026-08-01T14:15:04.693973Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T141504482999Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T141504482999Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f3bd90b8a9275d09684fcffd5fb938ffcdd03b3d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T141504482999Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8f5f327ba8c8b97c25a321905ab73827c03d99ea3acbe956e8049573833f92b +size 10002820 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T141400754097Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T141400754097Z.json new file mode 100644 index 0000000000000000000000000000000000000000..add707311384443ad7a9cee2fae395cf895805d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T141400754097Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 87698244608, + "active_bytes_peak_delta": 52231908352, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87698244608, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 86995437568, + "device": 3, + "inactive_split_bytes_current": 5458660352, + "inactive_split_bytes_peak": 9796535296, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 97196703744, + "reserved_bytes_peak": 97196703744, + "segment_count_current": 451, + "segment_count_peak": 451, + "timestamp_utc": "2026-08-01T14:14:00.753490Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 86995437568, + "allocated_bytes_peak_delta": 51529101312, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:12:52.066949Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5458660352, + "inactive_split_bytes_current_delta": 5325841408, + "inactive_split_bytes_peak": 9796535296, + "inactive_split_bytes_peak_delta": 9636149248, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 97196703744, + "reserved_bytes_current_delta": 57952698368, + "reserved_bytes_peak": 97196703744, + "reserved_bytes_peak_delta": 57952698368, + "segment_count_current": 451, + "segment_count_current_delta": 380, + "segment_count_peak": 451, + "segment_count_peak_delta": 380, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T141400754097Z.pickle", + "timestamp_utc": "2026-08-01T14:14:00.873795Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T141400754097Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T141400754097Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2275cd0431a7d37d3b84d3a61f8bb22314fe947a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T141400754097Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b98025ede62480289ffa53989c38efea68dc6a24cae5b7add36ac636062aadc6 +size 5955310 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T141402053135Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T141402053135Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7c8670e8a3af61417ca1fbf18a55b90d717de8a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T141402053135Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110976802816, + "active_bytes_peak_delta": 23278558208, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110976802816, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110624481280, + "device": 3, + "inactive_split_bytes_current": 8103169024, + "inactive_split_bytes_peak": 11006739456, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116222066688, + "reserved_bytes_peak": 116222066688, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:02.052721Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110624481280, + "allocated_bytes_peak_delta": 23629043712, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87698244608, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 86995437568, + "device": 3, + "inactive_split_bytes_current": 5458660352, + "inactive_split_bytes_peak": 9796535296, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 97196703744, + "reserved_bytes_peak": 97196703744, + "segment_count_current": 451, + "segment_count_peak": 451, + "timestamp_utc": "2026-08-01T14:14:00.890838Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8103169024, + "inactive_split_bytes_current_delta": 2644508672, + "inactive_split_bytes_peak": 11006739456, + "inactive_split_bytes_peak_delta": 1210204160, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116222066688, + "reserved_bytes_current_delta": 19025362944, + "reserved_bytes_peak": 116222066688, + "reserved_bytes_peak_delta": 19025362944, + "segment_count_current": 530, + "segment_count_current_delta": 79, + "segment_count_peak": 530, + "segment_count_peak_delta": 79, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T141402053135Z.pickle", + "timestamp_utc": "2026-08-01T14:14:02.169054Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T141402053135Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T141402053135Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2c6941e473f5796b9ac508aa178977441679b103 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T141402053135Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18955b8fe49bdb27d7a681aa96f4cf2a47ed04860a93fbc09fc737c56caa6375 +size 6366555 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T141403334333Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T141403334333Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b725f241f46e562cf2ebe26d348c06d6f87681a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T141403334333Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111328186368, + "active_bytes_peak_delta": 351383552, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111328186368, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110624861184, + "device": 3, + "inactive_split_bytes_current": 7807470592, + "inactive_split_bytes_peak": 11062509568, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117046247424, + "reserved_bytes_peak": 117046247424, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:03.333933Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110624861184, + "allocated_bytes_peak_delta": 379904, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110976802816, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110624481280, + "device": 3, + "inactive_split_bytes_current": 8103169024, + "inactive_split_bytes_peak": 11006739456, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116222066688, + "reserved_bytes_peak": 116222066688, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:02.183290Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7807470592, + "inactive_split_bytes_current_delta": -295698432, + "inactive_split_bytes_peak": 11062509568, + "inactive_split_bytes_peak_delta": 55770112, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117046247424, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117046247424, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 531, + "segment_count_current_delta": 1, + "segment_count_peak": 531, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T141403334333Z.pickle", + "timestamp_utc": "2026-08-01T14:14:03.452425Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T141403334333Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T141403334333Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eae8274a03b43f36aae936f2c6c7f315179e270c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T141403334333Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dad197c2e960b3dc7674779848661a3bcb6f9f5f8920fbfeb724a9f6708bb1ea +size 6116922 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T141404630653Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T141404630653Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15c080e5556fe66febfedcde80b7faf36406be3a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T141404630653Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111328186368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111328186368, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627192832, + "device": 3, + "inactive_split_bytes_current": 8455490560, + "inactive_split_bytes_peak": 12707843072, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:04.630254Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627192832, + "allocated_bytes_peak_delta": 2331648, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111328186368, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110624861184, + "device": 3, + "inactive_split_bytes_current": 7807470592, + "inactive_split_bytes_peak": 11062509568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117046247424, + "reserved_bytes_peak": 117046247424, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:03.468715Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8455490560, + "inactive_split_bytes_current_delta": 648019968, + "inactive_split_bytes_peak": 12707843072, + "inactive_split_bytes_peak_delta": 1645333504, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117870428160, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117870428160, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 532, + "segment_count_current_delta": 1, + "segment_count_peak": 532, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T141404630653Z.pickle", + "timestamp_utc": "2026-08-01T14:14:04.740556Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T141404630653Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T141404630653Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ef99288bc664fc6f137b557fbc333fe696b2085f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T141404630653Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:deb7f7a89d433d3d5f119150ee68f310bff7799aa935f13ebdf97ff221961f66 +size 6116195 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T141406196651Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T141406196651Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d831f8a00a95fdfba4938576a3aa3cb217d35b76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T141406196651Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111331767296, + "active_bytes_peak_delta": 3580928, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111331767296, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627193856, + "device": 3, + "inactive_split_bytes_current": 7985728512, + "inactive_split_bytes_peak": 12707843072, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:06.196241Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110627193856, + "allocated_bytes_peak_delta": 1024, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111328186368, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627192832, + "device": 3, + "inactive_split_bytes_current": 8455490560, + "inactive_split_bytes_peak": 12707843072, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:04.754393Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7985728512, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 12707843072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117870428160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117870428160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T141406196651Z.pickle", + "timestamp_utc": "2026-08-01T14:14:06.304403Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T141406196651Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T141406196651Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c50961c98627947aabede40a70cc1babd215a2e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T141406196651Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:333558c16b0ddcf477df453700ce48a114ba5aede4e3b43cf1afd5a6eb703335 +size 6638595 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T141407489998Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T141407489998Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bb7891048743f961156c7fe40a5551c8fc8b6012 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T141407489998Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111334747136, + "active_bytes_peak_delta": 2979840, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111334747136, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110630104064, + "device": 3, + "inactive_split_bytes_current": 8394673152, + "inactive_split_bytes_peak": 12707843072, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:07.489594Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110630104064, + "allocated_bytes_peak_delta": 2910208, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111331767296, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110627193856, + "device": 3, + "inactive_split_bytes_current": 7985728512, + "inactive_split_bytes_peak": 12707843072, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:06.317900Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8394673152, + "inactive_split_bytes_current_delta": 408944640, + "inactive_split_bytes_peak": 12707843072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117870428160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117870428160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T141407489998Z.pickle", + "timestamp_utc": "2026-08-01T14:14:07.595090Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T141407489998Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T141407489998Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d27f9823120a4494b0c40d2121b170a05fd95a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T141407489998Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:307d46a5b889cfde81ee33afbc0db52132d0b806312e32a7ee83a817a9008828 +size 6638371 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T141408787232Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T141408787232Z.json new file mode 100644 index 0000000000000000000000000000000000000000..02ce733c597963a7ef168cf22095e1b91a9134bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T141408787232Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111336222720, + "active_bytes_peak_delta": 1475584, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336222720, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110631649280, + "device": 3, + "inactive_split_bytes_current": 7159450624, + "inactive_split_bytes_peak": 12707843072, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:08.786822Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110631649280, + "allocated_bytes_peak_delta": 1545216, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111334747136, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110630104064, + "device": 3, + "inactive_split_bytes_current": 8394673152, + "inactive_split_bytes_peak": 12707843072, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:07.608317Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7159450624, + "inactive_split_bytes_current_delta": -1235222528, + "inactive_split_bytes_peak": 12707843072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117870428160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117870428160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T141408787232Z.pickle", + "timestamp_utc": "2026-08-01T14:14:08.894086Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T141408787232Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T141408787232Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fdfef16fb1243b1197cfab40e53c990a87b8779d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T141408787232Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa62dc330930520758cad84142cd925d576c4cd9c2f5cfa6c0d14220a19b21e0 +size 6637826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T141410231417Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T141410231417Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db118b821b10ab608d00d2e76e425aeca5654119 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T141410231417Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111336222720, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336222720, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110631649280, + "device": 3, + "inactive_split_bytes_current": 8042351616, + "inactive_split_bytes_peak": 12707843072, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:10.230989Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110631649280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336222720, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110631649280, + "device": 3, + "inactive_split_bytes_current": 7159450624, + "inactive_split_bytes_peak": 12707843072, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:08.907564Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8042351616, + "inactive_split_bytes_current_delta": 882900992, + "inactive_split_bytes_peak": 12707843072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117870428160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117870428160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T141410231417Z.pickle", + "timestamp_utc": "2026-08-01T14:14:10.338034Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T141410231417Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T141410231417Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f1ba9852f0e5550517582d098887b456fffb24a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T141410231417Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e96fe211454a63d405085b596cf354ff0cd79ee90a5477d5359a421776e81d1f +size 6637644 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T141411517629Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T141411517629Z.json new file mode 100644 index 0000000000000000000000000000000000000000..af455d26f4548c7a42a15ae823bddb6f9d36170c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T141411517629Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 111336222720, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 111336222720, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 110631649280, + "device": 3, + "inactive_split_bytes_current": 7571540992, + "inactive_split_bytes_peak": 12707843072, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:11.517159Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 110631649280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336222720, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110631649280, + "device": 3, + "inactive_split_bytes_current": 8042351616, + "inactive_split_bytes_peak": 12707843072, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:10.351467Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7571540992, + "inactive_split_bytes_current_delta": -470810624, + "inactive_split_bytes_peak": 12707843072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117870428160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117870428160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T141411517629Z.pickle", + "timestamp_utc": "2026-08-01T14:14:11.620870Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T141411517629Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T141411517629Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76797ad6135fc2d071ed4bc410827744586ee704 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T141411517629Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce90b1d26334ccd7c3f1f674a840d7420cf2f18a282f4f88fa1aa5b255c2c674 +size 6637608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T141504482996Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T141504482996Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2998dc5b52d44e862ed3d07fef2c42742e5a2d88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T141504482996Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11275337728, + "active_bytes_peak": 111336222720, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 111336222720, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 110631649280, + "device": 3, + "inactive_split_bytes_current": 7394331648, + "inactive_split_bytes_peak": 12707843072, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:15:04.482522Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11275337728, + "allocated_bytes_peak": 110631649280, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 111336222720, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 110631649280, + "device": 3, + "inactive_split_bytes_current": 7571540992, + "inactive_split_bytes_peak": 12707843072, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117870428160, + "reserved_bytes_peak": 117870428160, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:11.634147Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7394331648, + "inactive_split_bytes_current_delta": -177209344, + "inactive_split_bytes_peak": 12707843072, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117870428160, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117870428160, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T141504482996Z.pickle", + "timestamp_utc": "2026-08-01T14:15:04.692515Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T141504482996Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T141504482996Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aea5d7581a9a53951c6afadf73f1980a4c9bc4b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T141504482996Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc4fa892470ab1d046bc7594c3e4f4c9fc3577c1b9008fa9a4258989a7eef2a9 +size 10534253 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T141400754071Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T141400754071Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bf7abd412a46370b0944c048859e4881f373eb92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T141400754071Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441192448, + "active_bytes_current_delta": 37209294848, + "active_bytes_peak": 87696129024, + "active_bytes_peak_delta": 52229350400, + "after": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 87696129024, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 87003054080, + "device": 4, + "inactive_split_bytes_current": 5268774912, + "inactive_split_bytes_peak": 8340321280, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 97666465792, + "reserved_bytes_peak": 97666465792, + "segment_count_current": 453, + "segment_count_peak": 453, + "timestamp_utc": "2026-08-01T14:14:00.753448Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441192448, + "allocated_bytes_current_delta": 37209294848, + "allocated_bytes_peak": 87003054080, + "allocated_bytes_peak_delta": 51536275456, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:12:52.066380Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5268774912, + "inactive_split_bytes_current_delta": 5136398336, + "inactive_split_bytes_peak": 8340321280, + "inactive_split_bytes_peak_delta": 8186911744, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 97666465792, + "reserved_bytes_current_delta": 58422460416, + "reserved_bytes_peak": 97666465792, + "reserved_bytes_peak_delta": 58422460416, + "segment_count_current": 453, + "segment_count_current_delta": 382, + "segment_count_peak": 453, + "segment_count_peak_delta": 382, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T141400754071Z.pickle", + "timestamp_utc": "2026-08-01T14:14:00.872970Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T141400754071Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T141400754071Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b0395c7c33b80f50269c6ddf0d4587fccc2c6897 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T141400754071Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6d7041595b59c88b4029b0d37c89728db9e60e2c44d12b11716bd939c566df3 +size 5964365 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T141402053107Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T141402053107Z.json new file mode 100644 index 0000000000000000000000000000000000000000..19cc5f5e64eed45f406a1ff1581e6dd4e7950fbb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T141402053107Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441192448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110987988992, + "active_bytes_peak_delta": 23291859968, + "after": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 110987988992, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110635667456, + "device": 4, + "inactive_split_bytes_current": 6795501568, + "inactive_split_bytes_peak": 10696379392, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116154957824, + "reserved_bytes_peak": 116154957824, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T14:14:02.052666Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441192448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110635667456, + "allocated_bytes_peak_delta": 23632613376, + "before": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 87696129024, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 87003054080, + "device": 4, + "inactive_split_bytes_current": 5268774912, + "inactive_split_bytes_peak": 8340321280, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 97666465792, + "reserved_bytes_peak": 97666465792, + "segment_count_current": 453, + "segment_count_peak": 453, + "timestamp_utc": "2026-08-01T14:14:00.890028Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6795501568, + "inactive_split_bytes_current_delta": 1526726656, + "inactive_split_bytes_peak": 10696379392, + "inactive_split_bytes_peak_delta": 2356058112, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116154957824, + "reserved_bytes_current_delta": 18488492032, + "reserved_bytes_peak": 116154957824, + "reserved_bytes_peak_delta": 18488492032, + "segment_count_current": 529, + "segment_count_current_delta": 76, + "segment_count_peak": 529, + "segment_count_peak_delta": 76, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T141402053107Z.pickle", + "timestamp_utc": "2026-08-01T14:14:02.172562Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T141402053107Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T141402053107Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..169b35555a811d7625cd43e1dc800086aabc6164 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T141402053107Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4d3d98f51cde0ade949edf20be3757da083bf396ca6a2d121c928d2abd0d1c6 +size 6661013 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T141403334316Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T141403334316Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d0719cebcbf7eb3b81c77d59a4e4af0c74bcba2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T141403334316Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441192448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111337427456, + "active_bytes_peak_delta": 349438464, + "after": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111337427456, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110637241344, + "device": 4, + "inactive_split_bytes_current": 7091200000, + "inactive_split_bytes_peak": 10696379392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116154957824, + "reserved_bytes_peak": 116154957824, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T14:14:03.333884Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441192448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110637241344, + "allocated_bytes_peak_delta": 1573888, + "before": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 110987988992, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110635667456, + "device": 4, + "inactive_split_bytes_current": 6795501568, + "inactive_split_bytes_peak": 10696379392, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116154957824, + "reserved_bytes_peak": 116154957824, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T14:14:02.186633Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7091200000, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 10696379392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116154957824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116154957824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 529, + "segment_count_current_delta": 0, + "segment_count_peak": 529, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T141403334316Z.pickle", + "timestamp_utc": "2026-08-01T14:14:03.444570Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T141403334316Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T141403334316Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da66b63c4c1e867ce49056cba8f3533333001763 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T141403334316Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e74162ca64a553199cea1d76cd757abe9d367905cd001e8b55a9ebdc1cf26c95 +size 6656879 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T141404630606Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T141404630606Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69f7530bb9cca8af5426015f5707bf162b370add --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T141404630606Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441192448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111344243200, + "active_bytes_peak_delta": 6815744, + "after": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111344243200, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 6443180032, + "inactive_split_bytes_peak": 10696379392, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116154957824, + "reserved_bytes_peak": 116154957824, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T14:14:04.630174Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441192448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110645891072, + "allocated_bytes_peak_delta": 8649728, + "before": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111337427456, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110637241344, + "device": 4, + "inactive_split_bytes_current": 7091200000, + "inactive_split_bytes_peak": 10696379392, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116154957824, + "reserved_bytes_peak": 116154957824, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T14:14:03.460403Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6443180032, + "inactive_split_bytes_current_delta": -648019968, + "inactive_split_bytes_peak": 10696379392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116154957824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116154957824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 529, + "segment_count_current_delta": 0, + "segment_count_peak": 529, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T141404630606Z.pickle", + "timestamp_utc": "2026-08-01T14:14:04.740309Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T141404630606Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T141404630606Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ef340689fc597f978c2a1b8220a0f0f4e6e805a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T141404630606Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b011320ac8940e49455dec2c7e9f86203dc41f10b281b3558666c9a6f11c859a +size 6654989 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T141406196635Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T141406196635Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c4beea4efad48985f291a3b560d53e0cbfbc0633 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T141406196635Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441192448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111344243200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111344243200, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 7208640512, + "inactive_split_bytes_peak": 10696379392, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116154957824, + "reserved_bytes_peak": 116154957824, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T14:14:06.196212Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441192448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110645891072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111344243200, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 6443180032, + "inactive_split_bytes_peak": 10696379392, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116154957824, + "reserved_bytes_peak": 116154957824, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T14:14:04.754025Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7208640512, + "inactive_split_bytes_current_delta": 765460480, + "inactive_split_bytes_peak": 10696379392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116154957824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116154957824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 529, + "segment_count_current_delta": 0, + "segment_count_peak": 529, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T141406196635Z.pickle", + "timestamp_utc": "2026-08-01T14:14:06.305784Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T141406196635Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T141406196635Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..283a19f0eba54b7755b68b5c2fc4376c6ff12063 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T141406196635Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab130c01499ee4e78489e262e31aaa05d1d31d482f662c7770e3e5a3fab25520 +size 6655517 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T141407489993Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T141407489993Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e0ec29e6062219d23d54e7d5a984fe116c5b5256 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T141407489993Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441192448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111344243200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111344243200, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 8030724096, + "inactive_split_bytes_peak": 10696379392, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116979138560, + "reserved_bytes_peak": 116979138560, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:07.489563Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441192448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110645891072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111344243200, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 7208640512, + "inactive_split_bytes_peak": 10696379392, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116154957824, + "reserved_bytes_peak": 116154957824, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T14:14:06.319482Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8030724096, + "inactive_split_bytes_current_delta": 822083584, + "inactive_split_bytes_peak": 10696379392, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116979138560, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 116979138560, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 530, + "segment_count_current_delta": 1, + "segment_count_peak": 530, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T141407489993Z.pickle", + "timestamp_utc": "2026-08-01T14:14:07.596752Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T141407489993Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T141407489993Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..18e88f61870914e73025913883f31d4cae2f1122 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T141407489993Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16fb09ff1bc63b51b0c0687739c48baa56700c70bf5035a9bf7f007cae27184b +size 6655509 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T141408787226Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T141408787226Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bca1202b606b8c72272da788acfcc2912b30f9f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T141408787226Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441192448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111344243200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111344243200, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 7795843072, + "inactive_split_bytes_peak": 10989464576, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116979138560, + "reserved_bytes_peak": 116979138560, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:08.786787Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441192448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110645891072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111344243200, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 8030724096, + "inactive_split_bytes_peak": 10696379392, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116979138560, + "reserved_bytes_peak": 116979138560, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:07.610287Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7795843072, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 10989464576, + "inactive_split_bytes_peak_delta": 293085184, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116979138560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116979138560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T141408787226Z.pickle", + "timestamp_utc": "2026-08-01T14:14:08.891509Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T141408787226Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T141408787226Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1151c75b056af2e0c998404a000d663a138c377b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T141408787226Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d313a3c9d610561c0022617a7e230a422efed2e2f7463aed3abaef0b783db75 +size 6654880 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T141410231382Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T141410231382Z.json new file mode 100644 index 0000000000000000000000000000000000000000..978316d28794abb0271a9ee851037fa9e20adf9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T141410231382Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441192448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111344243200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111344243200, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 7265263616, + "inactive_split_bytes_peak": 11102710784, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116979138560, + "reserved_bytes_peak": 116979138560, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:10.230927Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441192448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110645891072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111344243200, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 7795843072, + "inactive_split_bytes_peak": 10989464576, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116979138560, + "reserved_bytes_peak": 116979138560, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:08.905029Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7265263616, + "inactive_split_bytes_current_delta": -530579456, + "inactive_split_bytes_peak": 11102710784, + "inactive_split_bytes_peak_delta": 113246208, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116979138560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116979138560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T141410231382Z.pickle", + "timestamp_utc": "2026-08-01T14:14:10.337417Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T141410231382Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T141410231382Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c8245a78217ef927bf2e1d9c36986bb5d49c1521 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T141410231382Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44be44850584e8deecb0b0036b7ea55bda24024da8ff9848abc0fb6354c5dba9 +size 6654099 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T141411517565Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T141411517565Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42e9481ab6f32b14be4e41aab01e517cdf1ed3a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T141411517565Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441192448, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111344243200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111344243200, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 6503997440, + "inactive_split_bytes_peak": 11107691520, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116979138560, + "reserved_bytes_peak": 116979138560, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:11.517124Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441192448, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110645891072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111344243200, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 7265263616, + "inactive_split_bytes_peak": 11102710784, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116979138560, + "reserved_bytes_peak": 116979138560, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:10.351107Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6503997440, + "inactive_split_bytes_current_delta": -761266176, + "inactive_split_bytes_peak": 11107691520, + "inactive_split_bytes_peak_delta": 4980736, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116979138560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116979138560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T141411517565Z.pickle", + "timestamp_utc": "2026-08-01T14:14:11.621782Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T141411517565Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T141411517565Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..471047ddeb5115648f01707c712bb625c27c68b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T141411517565Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:befd40a322006acb96a5049e1e4a371ea93305dee491691e70cee9e8b88c4352 +size 6653667 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T141504482986Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T141504482986Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7a7ef569d99464388445c9222f0dc6800cdb84d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T141504482986Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166903296, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 111347580928, + "active_bytes_peak_delta": 3337728, + "after": { + "active_bytes_current": 61166903296, + "active_bytes_peak": 111347580928, + "allocated_bytes_current": 61166903296, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 6204104704, + "inactive_split_bytes_peak": 11276307456, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116979138560, + "reserved_bytes_peak": 116979138560, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:15:04.482521Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166903296, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110645891072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441192448, + "active_bytes_peak": 111344243200, + "allocated_bytes_current": 72441192448, + "allocated_bytes_peak": 110645891072, + "device": 4, + "inactive_split_bytes_current": 6503997440, + "inactive_split_bytes_peak": 11107691520, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116979138560, + "reserved_bytes_peak": 116979138560, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:11.635140Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6204104704, + "inactive_split_bytes_current_delta": -299892736, + "inactive_split_bytes_peak": 11276307456, + "inactive_split_bytes_peak_delta": 168615936, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116979138560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116979138560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T141504482986Z.pickle", + "timestamp_utc": "2026-08-01T14:15:04.683787Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T141504482986Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T141504482986Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5296a34ceabb0a79cf896f9bb97533c4f0e47d3e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T141504482986Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b91aaea8b1c62687dfec95d6eca009676f15243d2951474eaa6a5fb631b5c9d8 +size 10556665 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T141400754059Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T141400754059Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d2ea976f0d0d471de2d3793cc4299c3c9bf7eee5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T141400754059Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 87351429120, + "active_bytes_peak_delta": 51885092864, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87351429120, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 86999107584, + "device": 5, + "inactive_split_bytes_current": 5987142656, + "inactive_split_bytes_peak": 9148777472, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 98371108864, + "reserved_bytes_peak": 98371108864, + "segment_count_current": 459, + "segment_count_peak": 459, + "timestamp_utc": "2026-08-01T14:14:00.753471Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 86999107584, + "allocated_bytes_peak_delta": 51532771328, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:12:52.067021Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5987142656, + "inactive_split_bytes_current_delta": 5854323712, + "inactive_split_bytes_peak": 9148777472, + "inactive_split_bytes_peak_delta": 8988391424, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 98371108864, + "reserved_bytes_current_delta": 59127103488, + "reserved_bytes_peak": 98371108864, + "reserved_bytes_peak_delta": 59127103488, + "segment_count_current": 459, + "segment_count_current_delta": 388, + "segment_count_peak": 459, + "segment_count_peak_delta": 388, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T141400754059Z.pickle", + "timestamp_utc": "2026-08-01T14:14:00.873063Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T141400754059Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T141400754059Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..19d226fa67cfbfaca39763ac477fa067d8caa794 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T141400754059Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcaf8c5227a9e24139262c99156c4a3d461a136e834ef993bb6ecec45367863e +size 5956722 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T141402053159Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T141402053159Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1c8529437e6c3a3eb808c3cac224dede40958e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T141402053159Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110977960960, + "active_bytes_peak_delta": 23626531840, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110977960960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625639424, + "device": 5, + "inactive_split_bytes_current": 8220609536, + "inactive_split_bytes_peak": 10893061120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116213678080, + "reserved_bytes_peak": 116213678080, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:02.052744Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625639424, + "allocated_bytes_peak_delta": 23626531840, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87351429120, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 86999107584, + "device": 5, + "inactive_split_bytes_current": 5987142656, + "inactive_split_bytes_peak": 9148777472, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 98371108864, + "reserved_bytes_peak": 98371108864, + "segment_count_current": 459, + "segment_count_peak": 459, + "timestamp_utc": "2026-08-01T14:14:00.894128Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8220609536, + "inactive_split_bytes_current_delta": 2233466880, + "inactive_split_bytes_peak": 10893061120, + "inactive_split_bytes_peak_delta": 1744283648, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116213678080, + "reserved_bytes_current_delta": 17842569216, + "reserved_bytes_peak": 116213678080, + "reserved_bytes_peak_delta": 17842569216, + "segment_count_current": 531, + "segment_count_current_delta": 72, + "segment_count_peak": 531, + "segment_count_peak_delta": 72, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T141402053159Z.pickle", + "timestamp_utc": "2026-08-01T14:14:02.171797Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T141402053159Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T141402053159Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d1b56145cb62c74c0588fa5ec35a1fd5abaa2db5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T141402053159Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a3d5ae42f7750abfddf49bdb082333ad5b7875279c7221193bdbbf05ed91870 +size 6119833 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T141403334404Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T141403334404Z.json new file mode 100644 index 0000000000000000000000000000000000000000..88d71deb65896f140f2a5e268d79305cc53a7edd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T141403334404Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111327318016, + "active_bytes_peak_delta": 349357056, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111327318016, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625639424, + "device": 5, + "inactive_split_bytes_current": 7278988288, + "inactive_split_bytes_peak": 11059717120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117037858816, + "reserved_bytes_peak": 117037858816, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:03.333996Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110625639424, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110977960960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625639424, + "device": 5, + "inactive_split_bytes_current": 8220609536, + "inactive_split_bytes_peak": 10893061120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116213678080, + "reserved_bytes_peak": 116213678080, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:02.188118Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7278988288, + "inactive_split_bytes_current_delta": -941621248, + "inactive_split_bytes_peak": 11059717120, + "inactive_split_bytes_peak_delta": 166656000, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117037858816, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117037858816, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 532, + "segment_count_current_delta": 1, + "segment_count_peak": 532, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T141403334404Z.pickle", + "timestamp_utc": "2026-08-01T14:14:03.444176Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T141403334404Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T141403334404Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f92b6882314dacc2d8b4057e9c53ebfd3928c50b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T141403334404Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3708d62c3677056414f9c778378d512cb290728afaac3418107091e98b3f730a +size 6115796 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T141404630654Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T141404630654Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c8701ef5773dbea7c873262e6e06e71bfb033bd7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T141404630654Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111331351040, + "active_bytes_peak_delta": 4033024, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111331351040, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110630620160, + "device": 5, + "inactive_split_bytes_current": 8572931072, + "inactive_split_bytes_peak": 12517024768, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117037858816, + "reserved_bytes_peak": 117037858816, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:04.630242Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110630620160, + "allocated_bytes_peak_delta": 4980736, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111327318016, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110625639424, + "device": 5, + "inactive_split_bytes_current": 7278988288, + "inactive_split_bytes_peak": 11059717120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117037858816, + "reserved_bytes_peak": 117037858816, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:03.460223Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8572931072, + "inactive_split_bytes_current_delta": 1293942784, + "inactive_split_bytes_peak": 12517024768, + "inactive_split_bytes_peak_delta": 1457307648, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117037858816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117037858816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T141404630654Z.pickle", + "timestamp_utc": "2026-08-01T14:14:04.742689Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T141404630654Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T141404630654Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f894421c4ed2cbbb6a9f710372f49ff668b7aba9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T141404630654Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f15567f33c98b9e9eab774f41cc14966d86a8bce47de1e65314198d6677c2bb9 +size 6638203 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T141406196671Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T141406196671Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8398478e0e8cafb4784ca55ed6012b86ba4151b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T141406196671Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111336361984, + "active_bytes_peak_delta": 5010944, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336361984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110631718912, + "device": 5, + "inactive_split_bytes_current": 7692127232, + "inactive_split_bytes_peak": 12517024768, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117037858816, + "reserved_bytes_peak": 117037858816, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:06.196273Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110631718912, + "allocated_bytes_peak_delta": 1098752, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111331351040, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110630620160, + "device": 5, + "inactive_split_bytes_current": 8572931072, + "inactive_split_bytes_peak": 12517024768, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117037858816, + "reserved_bytes_peak": 117037858816, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:04.758381Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7692127232, + "inactive_split_bytes_current_delta": -880803840, + "inactive_split_bytes_peak": 12517024768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117037858816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117037858816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T141406196671Z.pickle", + "timestamp_utc": "2026-08-01T14:14:06.307439Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T141406196671Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T141406196671Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..36bb534880c1da1d146060f78a8c61c3fedc2217 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T141406196671Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a96f1a5c63bd365e4512a8c9db554619d1bf54edcef9557594e9a216d6f6cc09 +size 6637033 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T141407490110Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T141407490110Z.json new file mode 100644 index 0000000000000000000000000000000000000000..051ceef44f645f1c70a7c5fe0833b967673b1023 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T141407490110Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111336361984, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336361984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110631718912, + "device": 5, + "inactive_split_bytes_current": 7805373440, + "inactive_split_bytes_peak": 12517024768, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117037858816, + "reserved_bytes_peak": 117037858816, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:07.489708Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110631718912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336361984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110631718912, + "device": 5, + "inactive_split_bytes_current": 7692127232, + "inactive_split_bytes_peak": 12517024768, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117037858816, + "reserved_bytes_peak": 117037858816, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:06.322923Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7805373440, + "inactive_split_bytes_current_delta": 113246208, + "inactive_split_bytes_peak": 12517024768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117037858816, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117037858816, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 532, + "segment_count_current_delta": 0, + "segment_count_peak": 532, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T141407490110Z.pickle", + "timestamp_utc": "2026-08-01T14:14:07.598176Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T141407490110Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T141407490110Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..96b05ef731677a9d59dc73fbb30eec52adeee6d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T141407490110Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aef50b2c9a52b22bd953da25abc4f47c6b8ddff2b47449d4ee9da920f9e87dc2 +size 6636692 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T141408787242Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T141408787242Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d94844fcb7290fffecc6d960cec27783cddc29ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T141408787242Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111336361984, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336361984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110631718912, + "device": 5, + "inactive_split_bytes_current": 7983631360, + "inactive_split_bytes_peak": 12517024768, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117862039552, + "reserved_bytes_peak": 117862039552, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:08.786827Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110631718912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336361984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110631718912, + "device": 5, + "inactive_split_bytes_current": 7805373440, + "inactive_split_bytes_peak": 12517024768, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117037858816, + "reserved_bytes_peak": 117037858816, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:07.613490Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7983631360, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 12517024768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117862039552, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117862039552, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 533, + "segment_count_current_delta": 1, + "segment_count_peak": 533, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T141408787242Z.pickle", + "timestamp_utc": "2026-08-01T14:14:08.901061Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T141408787242Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T141408787242Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..62c5dfb74b266c9f3cae81d051da8c2e6f47cb9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T141408787242Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f75ffac0265cd2089e039b6dd1f775397ed91041c9ba143e4952100f356e55ba +size 6113929 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T141410231528Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T141410231528Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9349fa4b90f11ac9848ad19fbe60dc2ee70aea0d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T141410231528Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111337139200, + "active_bytes_peak_delta": 777216, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337139200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110632565760, + "device": 5, + "inactive_split_bytes_current": 7809567744, + "inactive_split_bytes_peak": 12517024768, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117862039552, + "reserved_bytes_peak": 117862039552, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:10.231098Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110632565760, + "allocated_bytes_peak_delta": 846848, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111336361984, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110631718912, + "device": 5, + "inactive_split_bytes_current": 7983631360, + "inactive_split_bytes_peak": 12517024768, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117862039552, + "reserved_bytes_peak": 117862039552, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:08.918882Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7809567744, + "inactive_split_bytes_current_delta": -174063616, + "inactive_split_bytes_peak": 12517024768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117862039552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117862039552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T141410231528Z.pickle", + "timestamp_utc": "2026-08-01T14:14:10.341327Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T141410231528Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T141410231528Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3c525c7524184006a219c14b14b9756e4f94d971 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T141410231528Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54f9b00f6caa3a9d311056eca3ec08ffd31b784b0aea90d8a03f6de365600ae9 +size 6636376 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T141411517567Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T141411517567Z.json new file mode 100644 index 0000000000000000000000000000000000000000..47481630944157ff545cd44c444837750b16a3ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T141411517567Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111337139200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337139200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110632565760, + "device": 5, + "inactive_split_bytes_current": 7161547776, + "inactive_split_bytes_peak": 12517024768, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117862039552, + "reserved_bytes_peak": 117862039552, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:11.517172Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110632565760, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337139200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110632565760, + "device": 5, + "inactive_split_bytes_current": 7809567744, + "inactive_split_bytes_peak": 12517024768, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117862039552, + "reserved_bytes_peak": 117862039552, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:10.356754Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7161547776, + "inactive_split_bytes_current_delta": -648019968, + "inactive_split_bytes_peak": 12517024768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117862039552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117862039552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T141411517567Z.pickle", + "timestamp_utc": "2026-08-01T14:14:11.624778Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T141411517567Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T141411517567Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3dbeffafcde7e3c2e1cc0e8ae716b3eb9c9dc6a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T141411517567Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65fb169b7edd0706b211513ea2ce4f510ed85ab4a9766adbb2470223793700c5 +size 6636249 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T141504483083Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T141504483083Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c7fcddc87ee508c696cbb84588f146006d713d9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T141504483083Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 111337139200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 111337139200, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 110632565760, + "device": 5, + "inactive_split_bytes_current": 7218170880, + "inactive_split_bytes_peak": 12517024768, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117862039552, + "reserved_bytes_peak": 117862039552, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:15:04.482516Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110632565760, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111337139200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110632565760, + "device": 5, + "inactive_split_bytes_current": 7161547776, + "inactive_split_bytes_peak": 12517024768, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117862039552, + "reserved_bytes_peak": 117862039552, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:11.640201Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7218170880, + "inactive_split_bytes_current_delta": 56623104, + "inactive_split_bytes_peak": 12517024768, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117862039552, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117862039552, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T141504483083Z.pickle", + "timestamp_utc": "2026-08-01T14:15:04.685479Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T141504483083Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T141504483083Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e60c6c01b345db5768add8aa59773855269afc8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T141504483083Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e56b4540351187007bdef8ededeb4a056c56454c8b43c537c757848a69fa3229 +size 10001900 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T141400754083Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T141400754083Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61826ac03433fa9c4cc44338f07c69175c9c2f39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T141400754083Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 87698244608, + "active_bytes_peak_delta": 52231908352, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87698244608, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 86997534720, + "device": 6, + "inactive_split_bytes_current": 6280743936, + "inactive_split_bytes_peak": 9612104704, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98136227840, + "reserved_bytes_peak": 98136227840, + "segment_count_current": 454, + "segment_count_peak": 454, + "timestamp_utc": "2026-08-01T14:14:00.753486Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 86997534720, + "allocated_bytes_peak_delta": 51531198464, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:12:52.066976Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6280743936, + "inactive_split_bytes_current_delta": 6147924992, + "inactive_split_bytes_peak": 9612104704, + "inactive_split_bytes_peak_delta": 9451718656, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98136227840, + "reserved_bytes_current_delta": 58892222464, + "reserved_bytes_peak": 98136227840, + "reserved_bytes_peak_delta": 58892222464, + "segment_count_current": 454, + "segment_count_current_delta": 383, + "segment_count_peak": 454, + "segment_count_peak_delta": 383, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T141400754083Z.pickle", + "timestamp_utc": "2026-08-01T14:14:00.869728Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T141400754083Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T141400754083Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..505a283683e06c9f4526b60d68e25e756d3f955c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T141400754083Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16aaf71d793cd98dd9662aa770b5707c1c2a08ea8ad6033b603c35d1057af095 +size 5956020 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T141402053152Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T141402053152Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cef4cd8cd876812b2ed02a57b837f1e327540b17 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T141402053152Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110971415040, + "active_bytes_peak_delta": 23273170432, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110971415040, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110620446720, + "device": 6, + "inactive_split_bytes_current": 7337708544, + "inactive_split_bytes_peak": 10478564352, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116280786944, + "reserved_bytes_peak": 116280786944, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:02.052746Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110620446720, + "allocated_bytes_peak_delta": 23622912000, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 87698244608, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 86997534720, + "device": 6, + "inactive_split_bytes_current": 6280743936, + "inactive_split_bytes_peak": 9612104704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98136227840, + "reserved_bytes_peak": 98136227840, + "segment_count_current": 454, + "segment_count_peak": 454, + "timestamp_utc": "2026-08-01T14:14:00.888520Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7337708544, + "inactive_split_bytes_current_delta": 1056964608, + "inactive_split_bytes_peak": 10478564352, + "inactive_split_bytes_peak_delta": 866459648, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116280786944, + "reserved_bytes_current_delta": 18144559104, + "reserved_bytes_peak": 116280786944, + "reserved_bytes_peak_delta": 18144559104, + "segment_count_current": 531, + "segment_count_current_delta": 77, + "segment_count_peak": 531, + "segment_count_peak_delta": 77, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T141402053152Z.pickle", + "timestamp_utc": "2026-08-01T14:14:02.170807Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T141402053152Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T141402053152Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9204d7d3890dbc0bdc669aa8dde7df6afdc92443 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T141402053152Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f919a40ca87314eaf82ae9418310b106910abf7a49c6e137589ae158ab3ab7d4 +size 6120556 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T141403334390Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T141403334390Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7bc29fbf4555a1394d7882f30fb1713351598810 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T141403334390Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111330725888, + "active_bytes_peak_delta": 359310848, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110635174400, + "device": 6, + "inactive_split_bytes_current": 7515966464, + "inactive_split_bytes_peak": 10478564352, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116280786944, + "reserved_bytes_peak": 116280786944, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:03.333974Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110635174400, + "allocated_bytes_peak_delta": 14727680, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 110971415040, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110620446720, + "device": 6, + "inactive_split_bytes_current": 7337708544, + "inactive_split_bytes_peak": 10478564352, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116280786944, + "reserved_bytes_peak": 116280786944, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:02.185104Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7515966464, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 10478564352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116280786944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116280786944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 531, + "segment_count_current_delta": 0, + "segment_count_peak": 531, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T141403334390Z.pickle", + "timestamp_utc": "2026-08-01T14:14:03.442196Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T141403334390Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T141403334390Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..70b81115224d76691a11b06fefaecfa62e475925 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T141403334390Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8ab7141b6c1906103be5491088fddcea1cc034fbf7fa910ae08768a7003d7fe +size 6639366 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T141404630630Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T141404630630Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dc21ac4eddcab154548260a9f94ca05287a93fed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T141404630630Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111330725888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110635174400, + "device": 6, + "inactive_split_bytes_current": 7633406976, + "inactive_split_bytes_peak": 10478564352, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116280786944, + "reserved_bytes_peak": 116280786944, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:04.630223Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110635174400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110635174400, + "device": 6, + "inactive_split_bytes_current": 7515966464, + "inactive_split_bytes_peak": 10478564352, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116280786944, + "reserved_bytes_peak": 116280786944, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:03.455816Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7633406976, + "inactive_split_bytes_current_delta": 117440512, + "inactive_split_bytes_peak": 10478564352, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116280786944, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116280786944, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 531, + "segment_count_current_delta": 0, + "segment_count_peak": 531, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T141404630630Z.pickle", + "timestamp_utc": "2026-08-01T14:14:04.746186Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T141404630630Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T141404630630Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..772ec4f60d7a081ab76a44996f5956dc044f48b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T141404630630Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34696dc32a2a35f241d7c1fb95e620f8468105128bc429a759f0b6028df6c754 +size 6639226 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T141406196622Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T141406196622Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f4570de5b6e8c551d4d075d13d2d90ad4db3c128 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T141406196622Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111330725888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110635174400, + "device": 6, + "inactive_split_bytes_current": 7159450624, + "inactive_split_bytes_peak": 11244064768, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117104967680, + "reserved_bytes_peak": 117104967680, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:06.196210Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110635174400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110635174400, + "device": 6, + "inactive_split_bytes_current": 7633406976, + "inactive_split_bytes_peak": 10478564352, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116280786944, + "reserved_bytes_peak": 116280786944, + "segment_count_current": 531, + "segment_count_peak": 531, + "timestamp_utc": "2026-08-01T14:14:04.762524Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7159450624, + "inactive_split_bytes_current_delta": -473956352, + "inactive_split_bytes_peak": 11244064768, + "inactive_split_bytes_peak_delta": 765500416, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117104967680, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117104967680, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 532, + "segment_count_current_delta": 1, + "segment_count_peak": 532, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T141406196622Z.pickle", + "timestamp_utc": "2026-08-01T14:14:06.310953Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T141406196622Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T141406196622Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dea13decf05bc525ac8a1d033ec6614c93717231 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T141406196622Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90fee13c79fe8e00cde9896f20e86ef08af50a3022e533576f2acb67b1a18d0d +size 6114838 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T141407489992Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T141407489992Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5f5258b13ed7aa06d04cbfdeaf5a2a1957e257d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T141407489992Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111330725888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110635174400, + "device": 6, + "inactive_split_bytes_current": 7746653184, + "inactive_split_bytes_peak": 12192477184, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:07.489582Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110635174400, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110635174400, + "device": 6, + "inactive_split_bytes_current": 7159450624, + "inactive_split_bytes_peak": 11244064768, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117104967680, + "reserved_bytes_peak": 117104967680, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:14:06.324772Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7746653184, + "inactive_split_bytes_current_delta": 587202560, + "inactive_split_bytes_peak": 12192477184, + "inactive_split_bytes_peak_delta": 948412416, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117929148416, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117929148416, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 533, + "segment_count_current_delta": 1, + "segment_count_peak": 533, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T141407489992Z.pickle", + "timestamp_utc": "2026-08-01T14:14:07.594293Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T141407489992Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T141407489992Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2b0bde24947c4eafd4f6388910a50b801ede8571 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T141407489992Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0982c051a1c7c16568a24edba8be1a04061b283f7a6a7a9c86a78515f61ea7d1 +size 6114743 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T141408787289Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T141408787289Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8fc909f3f0ca0223a47f5430b4363982e303d09d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T141408787289Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111330725888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110637075456, + "device": 6, + "inactive_split_bytes_current": 7924911104, + "inactive_split_bytes_peak": 12192477184, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:08.786869Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110637075456, + "allocated_bytes_peak_delta": 1901056, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110635174400, + "device": 6, + "inactive_split_bytes_current": 7746653184, + "inactive_split_bytes_peak": 12192477184, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:07.607669Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7924911104, + "inactive_split_bytes_current_delta": 178257920, + "inactive_split_bytes_peak": 12192477184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117929148416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117929148416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T141408787289Z.pickle", + "timestamp_utc": "2026-08-01T14:14:08.893590Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T141408787289Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T141408787289Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b9ced9f4bdf1e2f0444d2c49178b316d91ac612 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T141408787289Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ded7125681be8c6bd530efb488b9a674b568003c76755ac88c8d2353850fe618 +size 6637405 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T141410231377Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T141410231377Z.json new file mode 100644 index 0000000000000000000000000000000000000000..44cd73be2af6fd6f21f9199303ef2786d422737c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T141410231377Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111330725888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110637075456, + "device": 6, + "inactive_split_bytes_current": 6807129088, + "inactive_split_bytes_peak": 12192477184, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:10.230954Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110637075456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110637075456, + "device": 6, + "inactive_split_bytes_current": 7924911104, + "inactive_split_bytes_peak": 12192477184, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:08.907054Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6807129088, + "inactive_split_bytes_current_delta": -1117782016, + "inactive_split_bytes_peak": 12192477184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117929148416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117929148416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T141410231377Z.pickle", + "timestamp_utc": "2026-08-01T14:14:10.337555Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T141410231377Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T141410231377Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2dd6d75e03e25cc8209c0681e42f605ca883c7c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T141410231377Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7dcd8dae90f12ea0df0f88c2cbc3e7fee433d923344c6f1885b7c9a119aad6e +size 6637140 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T141411517665Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T141411517665Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2227243dc09b67a716ef51753485b605606b0a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T141411517665Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111330725888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110637075456, + "device": 6, + "inactive_split_bytes_current": 7455149056, + "inactive_split_bytes_peak": 12192477184, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:11.517214Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110637075456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110637075456, + "device": 6, + "inactive_split_bytes_current": 6807129088, + "inactive_split_bytes_peak": 12192477184, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:10.351028Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7455149056, + "inactive_split_bytes_current_delta": 648019968, + "inactive_split_bytes_peak": 12192477184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117929148416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117929148416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T141411517665Z.pickle", + "timestamp_utc": "2026-08-01T14:14:11.624571Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T141411517665Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T141411517665Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..56ebdc3b608136d760f6a4355e6bd83dc43f75a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T141411517665Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:118ba7a6281f1155a8739274afc0576793bc80650f08d97e629b799751687548 +size 6636926 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T141504482981Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T141504482981Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3efed2ecf3345ebbeecec13e7dceadb493668fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T141504482981Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 111330725888, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 110637075456, + "device": 6, + "inactive_split_bytes_current": 6159109120, + "inactive_split_bytes_peak": 12192477184, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:15:04.482515Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110637075456, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 111330725888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 110637075456, + "device": 6, + "inactive_split_bytes_current": 7455149056, + "inactive_split_bytes_peak": 12192477184, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117929148416, + "reserved_bytes_peak": 117929148416, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:14:11.637855Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6159109120, + "inactive_split_bytes_current_delta": -1296039936, + "inactive_split_bytes_peak": 12192477184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117929148416, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117929148416, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T141504482981Z.pickle", + "timestamp_utc": "2026-08-01T14:15:04.693391Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T141504482981Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T141504482981Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2899a2d2711ab639a89764b89f192ceddf146542 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T141504482981Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6f99b4dcaaf3d3f1e865419864158f3396713e4edd567198c00861a34bf1722 +size 10002568 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T141400754102Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T141400754102Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36c95941585745de01f087bce4640063db90e49c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T141400754102Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 37208595968, + "active_bytes_peak": 87347496448, + "active_bytes_peak_delta": 51881160192, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 87347496448, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 86995174912, + "device": 7, + "inactive_split_bytes_current": 5458659840, + "inactive_split_bytes_peak": 10124433408, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 97196703744, + "reserved_bytes_peak": 97196703744, + "segment_count_current": 450, + "segment_count_peak": 450, + "timestamp_utc": "2026-08-01T14:14:00.753510Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 37208595968, + "allocated_bytes_peak": 86995174912, + "allocated_bytes_peak_delta": 51528838656, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:12:52.069377Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5458659840, + "inactive_split_bytes_current_delta": 5325840896, + "inactive_split_bytes_peak": 10124433408, + "inactive_split_bytes_peak_delta": 9964047360, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 97196703744, + "reserved_bytes_current_delta": 57952698368, + "reserved_bytes_peak": 97196703744, + "reserved_bytes_peak_delta": 57952698368, + "segment_count_current": 450, + "segment_count_current_delta": 379, + "segment_count_peak": 450, + "segment_count_peak_delta": 379, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T141400754102Z.pickle", + "timestamp_utc": "2026-08-01T14:14:00.869266Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T141400754102Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T141400754102Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..781625f9422f48e3caff40702acff82f479b038f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T141400754102Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33f26b7030fa05477a03abb50d97267ae6dc460041ff9fc70566d19eb993a511 +size 5956798 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T141402053113Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T141402053113Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3c706d37627ccfe9e73c130a724e2036ba6ee677 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T141402053113Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 110980746752, + "active_bytes_peak_delta": 23633250304, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110980746752, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110628425216, + "device": 7, + "inactive_split_bytes_current": 7924910592, + "inactive_split_bytes_peak": 12640309760, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116683440128, + "reserved_bytes_peak": 116683440128, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T14:14:02.052710Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110628425216, + "allocated_bytes_peak_delta": 23633250304, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 87347496448, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 86995174912, + "device": 7, + "inactive_split_bytes_current": 5458659840, + "inactive_split_bytes_peak": 10124433408, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 97196703744, + "reserved_bytes_peak": 97196703744, + "segment_count_current": 450, + "segment_count_peak": 450, + "timestamp_utc": "2026-08-01T14:14:00.887876Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7924910592, + "inactive_split_bytes_current_delta": 2466250752, + "inactive_split_bytes_peak": 12640309760, + "inactive_split_bytes_peak_delta": 2515876352, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116683440128, + "reserved_bytes_current_delta": 19486736384, + "reserved_bytes_peak": 116683440128, + "reserved_bytes_peak_delta": 19486736384, + "segment_count_current": 529, + "segment_count_current_delta": 79, + "segment_count_peak": 529, + "segment_count_peak_delta": 79, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T141402053113Z.pickle", + "timestamp_utc": "2026-08-01T14:14:02.168640Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T141402053113Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T141402053113Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c46d1897cf0339bc36702a7b33bf2f43e8247b5d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T141402053113Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:373822eed8532ca321845ec5b928ada997d55e89ecc6698107d0ef308cb551e4 +size 6121920 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T141403334313Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T141403334313Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b361f0faa03140f27cc68e6dd3b860302c567f76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T141403334313Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111329276416, + "active_bytes_peak_delta": 348529664, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111329276416, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110628425216, + "device": 7, + "inactive_split_bytes_current": 7515965952, + "inactive_split_bytes_peak": 12640309760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:03.333906Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110628425216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 110980746752, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110628425216, + "device": 7, + "inactive_split_bytes_current": 7924910592, + "inactive_split_bytes_peak": 12640309760, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116683440128, + "reserved_bytes_peak": 116683440128, + "segment_count_current": 529, + "segment_count_peak": 529, + "timestamp_utc": "2026-08-01T14:14:02.184832Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7515965952, + "inactive_split_bytes_current_delta": -408944640, + "inactive_split_bytes_peak": 12640309760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117507620864, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 117507620864, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 530, + "segment_count_current_delta": 1, + "segment_count_peak": 530, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T141403334313Z.pickle", + "timestamp_utc": "2026-08-01T14:14:03.444181Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T141403334313Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T141403334313Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60e6e0293fc48e445bd48b8a8347d17b6e52514e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T141403334313Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a971096be1b4791a808077cd05feab2b5bfdc72954349aca96cf4c05b3b505f +size 6117611 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T141404630666Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T141404630666Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5044641701fd715942bde5009540898dfec29236 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T141404630666Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111329276416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111329276416, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110628425216, + "device": 7, + "inactive_split_bytes_current": 7985728000, + "inactive_split_bytes_peak": 12640309760, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:04.630256Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110628425216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111329276416, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110628425216, + "device": 7, + "inactive_split_bytes_current": 7515965952, + "inactive_split_bytes_peak": 12640309760, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:03.460047Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7985728000, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 12640309760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117507620864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117507620864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T141404630666Z.pickle", + "timestamp_utc": "2026-08-01T14:14:04.741038Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T141404630666Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T141404630666Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a32506a69247960c8ae3776e8dd7ba8846dc75b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T141404630666Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60d998ad9d632219159b525d4a5dd064b23b637154889310f05ce7059b7d8198 +size 6640124 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T141406196629Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T141406196629Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a1753d9e25383b5feb127765f1432968b20a757e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T141406196629Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111334346240, + "active_bytes_peak_delta": 5069824, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111334346240, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110629703168, + "device": 7, + "inactive_split_bytes_current": 9042692608, + "inactive_split_bytes_peak": 12640309760, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:06.196217Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110629703168, + "allocated_bytes_peak_delta": 1277952, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111329276416, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110628425216, + "device": 7, + "inactive_split_bytes_current": 7985728000, + "inactive_split_bytes_peak": 12640309760, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:04.756670Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9042692608, + "inactive_split_bytes_current_delta": 1056964608, + "inactive_split_bytes_peak": 12640309760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117507620864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117507620864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T141406196629Z.pickle", + "timestamp_utc": "2026-08-01T14:14:06.305369Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T141406196629Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T141406196629Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8bdfe184db0dc5ab1a01288fe0146466bd65c025 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T141406196629Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a4d086765e6f56a595de74523d686061077cf579786f4269e2b108930cf2612 +size 6639788 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T141407489988Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T141407489988Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9041340363f98da29833615cbe5e3785813796cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T141407489988Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111334346240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111334346240, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110629703168, + "device": 7, + "inactive_split_bytes_current": 7985728000, + "inactive_split_bytes_peak": 12640309760, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:07.489595Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110629703168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111334346240, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110629703168, + "device": 7, + "inactive_split_bytes_current": 9042692608, + "inactive_split_bytes_peak": 12640309760, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:06.320849Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7985728000, + "inactive_split_bytes_current_delta": -1056964608, + "inactive_split_bytes_peak": 12640309760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117507620864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117507620864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T141407489988Z.pickle", + "timestamp_utc": "2026-08-01T14:14:07.608124Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T141407489988Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T141407489988Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c129d44ccf1b9b36d8841c951ec3f887e29a7e0d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T141407489988Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3511e15e1125c9854a80dde820902122548d7f436b719cb3f43c6ca82fb1573 +size 6639345 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T141408787308Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T141408787308Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5b9b2f6650c9a560df246fb2a55b4174bc2db64d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T141408787308Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111334346240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111334346240, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110629703168, + "device": 7, + "inactive_split_bytes_current": 7633406464, + "inactive_split_bytes_peak": 12640309760, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:08.786894Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110629703168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111334346240, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110629703168, + "device": 7, + "inactive_split_bytes_current": 7985728000, + "inactive_split_bytes_peak": 12640309760, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:07.626963Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7633406464, + "inactive_split_bytes_current_delta": -352321536, + "inactive_split_bytes_peak": 12640309760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117507620864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117507620864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T141408787308Z.pickle", + "timestamp_utc": "2026-08-01T14:14:08.896362Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T141408787308Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T141408787308Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e2fa0cbfce4c63bea1f5dd0c97f64755e079f27 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T141408787308Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a2b75d692026037153c39456bf14eb4f5965712eb0b657d92d657ea1d4ecf59 +size 6639250 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T141410231359Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T141410231359Z.json new file mode 100644 index 0000000000000000000000000000000000000000..21aa258c04594db113948cdf917611bad4819cf6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T141410231359Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111334346240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111334346240, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110629703168, + "device": 7, + "inactive_split_bytes_current": 7924910592, + "inactive_split_bytes_peak": 12640309760, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:10.230949Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110629703168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111334346240, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110629703168, + "device": 7, + "inactive_split_bytes_current": 7633406464, + "inactive_split_bytes_peak": 12640309760, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:08.911734Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7924910592, + "inactive_split_bytes_current_delta": 291504128, + "inactive_split_bytes_peak": 12640309760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117507620864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117507620864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T141410231359Z.pickle", + "timestamp_utc": "2026-08-01T14:14:10.343136Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T141410231359Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T141410231359Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..81196e528c42c95e064153639a5c9408dc8400cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T141410231359Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1774cc1eed6b20f51430d1584d38383f0869dd146609a0587baa11c52388eae9 +size 6639143 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T141411517560Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T141411517560Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a91028a5c97f9f3c1c37c95ba2b381b1a9b97370 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T141411517560Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 111334346240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111334346240, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110629703168, + "device": 7, + "inactive_split_bytes_current": 7985728000, + "inactive_split_bytes_peak": 12640309760, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:11.517168Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 110629703168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111334346240, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110629703168, + "device": 7, + "inactive_split_bytes_current": 7924910592, + "inactive_split_bytes_peak": 12640309760, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:10.358684Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7985728000, + "inactive_split_bytes_current_delta": 60817408, + "inactive_split_bytes_peak": 12640309760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117507620864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117507620864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T141411517560Z.pickle", + "timestamp_utc": "2026-08-01T14:14:11.623197Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T141411517560Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T141411517560Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5211e175a87071ad2f97ada9e24cd34a305dfa04 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T141411517560Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5da6dc94e57c2a7c28f2a237abff50e622a348bf60ac565094dc038dca30d178 +size 6638951 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T141504482969Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T141504482969Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7b7fe01a0cc6e0438911c2ad42d171710ad326a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T141504482969Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165762048, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 111334346240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165762048, + "active_bytes_peak": 111334346240, + "allocated_bytes_current": 61165762048, + "allocated_bytes_peak": 110629703168, + "device": 7, + "inactive_split_bytes_current": 8159791616, + "inactive_split_bytes_peak": 12640309760, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:15:04.482518Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165762048, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 110629703168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 111334346240, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 110629703168, + "device": 7, + "inactive_split_bytes_current": 7985728000, + "inactive_split_bytes_peak": 12640309760, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117507620864, + "reserved_bytes_peak": 117507620864, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:14:11.638610Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8159791616, + "inactive_split_bytes_current_delta": 174063616, + "inactive_split_bytes_peak": 12640309760, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117507620864, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117507620864, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T141504482969Z.pickle", + "timestamp_utc": "2026-08-01T14:15:04.685533Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T141504482969Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T141504482969Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..725b1ac95ffcda89ff4587498cb1ac11cdabac2b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T141504482969Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa7eb0e36d06d5609502cf9e8e4a7baf09017870a103ad05f020e528734112d8 +size 10004582 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..0204d459b2ae0cc628234919c60d1ec2815d1536 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 4096 +capacity: 1.25 +expert_dispatch_overlap: off +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6040 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 226 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T141751322497Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T141751322497Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2638e4a3e48441f1ba5a67002c4de0688572fd3d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T141751322497Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854818816, + "active_bytes_current_delta": 31623363584, + "active_bytes_peak": 89627477504, + "active_bytes_peak_delta": 54161141248, + "after": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 89627477504, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 89627477504, + "device": 0, + "inactive_split_bytes_current": 9609440256, + "inactive_split_bytes_peak": 10699781120, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 95982452736, + "reserved_bytes_peak": 95982452736, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T14:17:51.322071Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854818816, + "allocated_bytes_current_delta": 31623363584, + "allocated_bytes_peak": 89627477504, + "allocated_bytes_peak_delta": 54161141248, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T14:16:43.029178Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9609440256, + "inactive_split_bytes_current_delta": 9474524160, + "inactive_split_bytes_peak": 10699781120, + "inactive_split_bytes_peak_delta": 10539149312, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 95982452736, + "reserved_bytes_current_delta": 56736350208, + "reserved_bytes_peak": 95982452736, + "reserved_bytes_peak_delta": 56736350208, + "segment_count_current": 467, + "segment_count_current_delta": 395, + "segment_count_peak": 467, + "segment_count_peak_delta": 395, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T141751322497Z.pickle", + "timestamp_utc": "2026-08-01T14:17:51.435504Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T141751322497Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T141751322497Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b08e1cdb25b0bf49280d9138b80563489befa556 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T141751322497Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d49f09c67b4ba1ccc4c2f50828d705e469c73f3c51b9beed60369e45a65994af +size 5457996 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T141752566001Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T141752566001Z.json new file mode 100644 index 0000000000000000000000000000000000000000..296ae6378acc00dc8cdf49fd84e397c01b50b68d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T141752566001Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854818816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113254673408, + "active_bytes_peak_delta": 23627195904, + "after": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113254673408, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113254673408, + "device": 0, + "inactive_split_bytes_current": 10330860544, + "inactive_split_bytes_peak": 12040643584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115001524224, + "reserved_bytes_peak": 115001524224, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:17:52.565477Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854818816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113254673408, + "allocated_bytes_peak_delta": 23627195904, + "before": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 89627477504, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 89627477504, + "device": 0, + "inactive_split_bytes_current": 9609440256, + "inactive_split_bytes_peak": 10699781120, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 95982452736, + "reserved_bytes_peak": 95982452736, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T14:17:51.456833Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10330860544, + "inactive_split_bytes_current_delta": 721420288, + "inactive_split_bytes_peak": 12040643584, + "inactive_split_bytes_peak_delta": 1340862464, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115001524224, + "reserved_bytes_current_delta": 19019071488, + "reserved_bytes_peak": 115001524224, + "reserved_bytes_peak_delta": 19019071488, + "segment_count_current": 532, + "segment_count_current_delta": 65, + "segment_count_peak": 532, + "segment_count_peak_delta": 65, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T141752566001Z.pickle", + "timestamp_utc": "2026-08-01T14:17:52.688782Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T141752566001Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T141752566001Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc16a8d5837447b63a6d0352ff9a448c230ea58e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T141752566001Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c498cb133f1310bd164c88aac70f057957291d778923e4ccbab46e9829034d8 +size 5602191 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T141753784494Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T141753784494Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6545bb5775bbcf357301ff4e14a89272adbd236b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T141753784494Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854818816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261218816, + "active_bytes_peak_delta": 6545408, + "after": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113261218816, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113261218816, + "device": 0, + "inactive_split_bytes_current": 10951617536, + "inactive_split_bytes_peak": 12040643584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:17:53.783900Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854818816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261218816, + "allocated_bytes_peak_delta": 6545408, + "before": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113254673408, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113254673408, + "device": 0, + "inactive_split_bytes_current": 10330860544, + "inactive_split_bytes_peak": 12040643584, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115001524224, + "reserved_bytes_peak": 115001524224, + "segment_count_current": 532, + "segment_count_peak": 532, + "timestamp_utc": "2026-08-01T14:17:52.717113Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10951617536, + "inactive_split_bytes_current_delta": 620756992, + "inactive_split_bytes_peak": 12040643584, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 115825704960, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 533, + "segment_count_current_delta": 1, + "segment_count_peak": 533, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T141753784494Z.pickle", + "timestamp_utc": "2026-08-01T14:17:53.892346Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T141753784494Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T141753784494Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..54843808025b09e888b30de8d8cd11dc9ee6e726 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T141753784494Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef927ac9768bc6f99539c15121f7b466d93b9cb11d489083349efe82b78a564a +size 5605355 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T141754980827Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T141754980827Z.json new file mode 100644 index 0000000000000000000000000000000000000000..79d26e0418741d87a9411c92400e020621accf7c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T141754980827Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854818816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261218816, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113261218816, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113261218816, + "device": 0, + "inactive_split_bytes_current": 11331202048, + "inactive_split_bytes_peak": 12155828224, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:17:54.980245Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854818816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261218816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113261218816, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113261218816, + "device": 0, + "inactive_split_bytes_current": 10951617536, + "inactive_split_bytes_peak": 12040643584, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:17:53.907597Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11331202048, + "inactive_split_bytes_current_delta": 379584512, + "inactive_split_bytes_peak": 12155828224, + "inactive_split_bytes_peak_delta": 115184640, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115825704960, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T141754980827Z.pickle", + "timestamp_utc": "2026-08-01T14:17:55.091352Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T141754980827Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T141754980827Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aaee512c85dfbd849158d354a91580453db26791 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T141754980827Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b417d1b8bd8c86b73646f5fcd5437d80af923a188c27fd4041661ea8bda6f637 +size 5602117 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T141756762442Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T141756762442Z.json new file mode 100644 index 0000000000000000000000000000000000000000..701e6b5e79d178443a58af32142251640321b691 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T141756762442Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854818816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261218816, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113261218816, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113261218816, + "device": 0, + "inactive_split_bytes_current": 10890800128, + "inactive_split_bytes_peak": 12155828224, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:17:56.761965Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854818816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261218816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113261218816, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113261218816, + "device": 0, + "inactive_split_bytes_current": 11331202048, + "inactive_split_bytes_peak": 12155828224, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:17:55.106668Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10890800128, + "inactive_split_bytes_current_delta": -440401920, + "inactive_split_bytes_peak": 12155828224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115825704960, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T141756762442Z.pickle", + "timestamp_utc": "2026-08-01T14:17:56.893312Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T141756762442Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T141756762442Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1be3fcb270b546d5268cd3632d4a4dd4b6c5e613 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T141756762442Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:662a206d6b6496b04eecb320aeaa1d45d6455e84e4682b64e1ee6076cf70c11e +size 5601716 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T141757966102Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T141757966102Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69c59563526580f83cd4b5649137bd99e0102fb0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T141757966102Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854818816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261218816, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113261218816, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113261218816, + "device": 0, + "inactive_split_bytes_current": 12335737856, + "inactive_split_bytes_peak": 12620218368, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:17:57.965679Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854818816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261218816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113261218816, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113261218816, + "device": 0, + "inactive_split_bytes_current": 10890800128, + "inactive_split_bytes_peak": 12155828224, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:17:56.913389Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 12335737856, + "inactive_split_bytes_current_delta": 1444937728, + "inactive_split_bytes_peak": 12620218368, + "inactive_split_bytes_peak_delta": 464390144, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115825704960, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T141757966102Z.pickle", + "timestamp_utc": "2026-08-01T14:17:58.064295Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T141757966102Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T141757966102Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9c335b8f0e8b6d3eaca244d5d9d1adf086809edc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T141757966102Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:835ee38893150132c1695d0034c307c4a7af9009b2e7dc73e18dcd67425f7bc6 +size 5604750 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T141759128841Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T141759128841Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aca5a5c09bf4d9d3ec40b74a08e5629f739ff1cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T141759128841Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854818816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261500416, + "active_bytes_peak_delta": 281600, + "after": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113261500416, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113261500416, + "device": 0, + "inactive_split_bytes_current": 10800622592, + "inactive_split_bytes_peak": 12620218368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:17:59.128417Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854818816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261500416, + "allocated_bytes_peak_delta": 281600, + "before": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113261218816, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113261218816, + "device": 0, + "inactive_split_bytes_current": 12335737856, + "inactive_split_bytes_peak": 12620218368, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:17:58.077141Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10800622592, + "inactive_split_bytes_current_delta": -1535115264, + "inactive_split_bytes_peak": 12620218368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115825704960, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T141759128841Z.pickle", + "timestamp_utc": "2026-08-01T14:17:59.230012Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T141759128841Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T141759128841Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..98332b6fa5a4ce453d2e8fe228704aabfd9c1352 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T141759128841Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df982bca44211fb135339bcbe1b274766a8917bd720d549df906332e694fd5f0 +size 5603970 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T141800314037Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T141800314037Z.json new file mode 100644 index 0000000000000000000000000000000000000000..09ae026e1adf85fe82489a7907cfb054b26c1f9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T141800314037Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854818816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113263363072, + "active_bytes_peak_delta": 1862656, + "after": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113263363072, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113263363072, + "device": 0, + "inactive_split_bytes_current": 11475905536, + "inactive_split_bytes_peak": 12620218368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:18:00.313508Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854818816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113263363072, + "allocated_bytes_peak_delta": 1862656, + "before": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113261500416, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113261500416, + "device": 0, + "inactive_split_bytes_current": 10800622592, + "inactive_split_bytes_peak": 12620218368, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:17:59.252748Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11475905536, + "inactive_split_bytes_current_delta": 675282944, + "inactive_split_bytes_peak": 12620218368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115825704960, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T141800314037Z.pickle", + "timestamp_utc": "2026-08-01T14:18:00.414045Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T141800314037Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T141800314037Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..81afcdfbec2a07646ea95dbc2d9c60c0a4ba83e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T141800314037Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d94e88c05015cf8dfa7435c02a06291a9c113184f8e04f4b0008b7df5fbfcb93 +size 5601416 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T141801477705Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T141801477705Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7d52f45be526a63109b07b17162b26830092ff15 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T141801477705Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854818816, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113263363072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113263363072, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113263363072, + "device": 0, + "inactive_split_bytes_current": 11775798272, + "inactive_split_bytes_peak": 12620218368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:18:01.477215Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854818816, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113263363072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113263363072, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113263363072, + "device": 0, + "inactive_split_bytes_current": 11475905536, + "inactive_split_bytes_peak": 12620218368, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:18:00.426916Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11775798272, + "inactive_split_bytes_current_delta": 299892736, + "inactive_split_bytes_peak": 12620218368, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115825704960, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T141801477705Z.pickle", + "timestamp_utc": "2026-08-01T14:18:01.578748Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T141801477705Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T141801477705Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c3b277cff32e619b73c7e0e26bb2e6855665f702 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T141801477705Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:286c826e0fe70fc5cea23277a17f21a79badf9a730ef1936e97fa25a2a754de4 +size 5604523 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T141857701374Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T141857701374Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ba2284dbbbaacefb89b84a418900910a49cc751 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T141857701374Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804175872, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113263363072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66804175872, + "active_bytes_peak": 113263363072, + "allocated_bytes_current": 66804175872, + "allocated_bytes_peak": 113263363072, + "device": 0, + "inactive_split_bytes_current": 9725094912, + "inactive_split_bytes_peak": 12668047360, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:18:57.700833Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804175872, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113263363072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854818816, + "active_bytes_peak": 113263363072, + "allocated_bytes_current": 66854818816, + "allocated_bytes_peak": 113263363072, + "device": 0, + "inactive_split_bytes_current": 11775798272, + "inactive_split_bytes_peak": 12620218368, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_peak": 115825704960, + "segment_count_current": 533, + "segment_count_peak": 533, + "timestamp_utc": "2026-08-01T14:18:01.591850Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9725094912, + "inactive_split_bytes_current_delta": -2050703360, + "inactive_split_bytes_peak": 12668047360, + "inactive_split_bytes_peak_delta": 47828992, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 115825704960, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115825704960, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 533, + "segment_count_current_delta": 0, + "segment_count_peak": 533, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T141857701374Z.pickle", + "timestamp_utc": "2026-08-01T14:18:57.928092Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T141857701374Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T141857701374Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1a99f822072393b80123e408b4595724bf1ad827 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T141857701374Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fca4d32cb4ef785bd15748bcac781132fa084965b542f79fa7f3c18a6dee987 +size 9214769 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T141751322478Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T141751322478Z.json new file mode 100644 index 0000000000000000000000000000000000000000..abc41708b426d1002fc0bdd507587dec0deea3c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T141751322478Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 89629266944, + "active_bytes_peak_delta": 54162930688, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89629266944, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89629266944, + "device": 1, + "inactive_split_bytes_current": 8870415360, + "inactive_split_bytes_peak": 11623535616, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 94701092864, + "reserved_bytes_peak": 94701092864, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:17:51.321897Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 89629266944, + "allocated_bytes_peak_delta": 54162930688, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:16:43.026139Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8870415360, + "inactive_split_bytes_current_delta": 8737596416, + "inactive_split_bytes_peak": 11623535616, + "inactive_split_bytes_peak_delta": 11463149568, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 94701092864, + "reserved_bytes_current_delta": 55457087488, + "reserved_bytes_peak": 94701092864, + "reserved_bytes_peak_delta": 55457087488, + "segment_count_current": 463, + "segment_count_current_delta": 392, + "segment_count_peak": 463, + "segment_count_peak_delta": 392, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T141751322478Z.pickle", + "timestamp_utc": "2026-08-01T14:17:51.430732Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T141751322478Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T141751322478Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de8036b2df1d57caa3d18d882f2893db35fb3957 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T141751322478Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:758dc07587474e31f65bb286336a1288ed0c97186f41481715c91e72bd09d1d4 +size 5445052 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T141752565783Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T141752565783Z.json new file mode 100644 index 0000000000000000000000000000000000000000..388299711b1dd54093a8f77902b3ed1f82d5eb68 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T141752565783Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113252524544, + "active_bytes_peak_delta": 23623257600, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113252524544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113252524544, + "device": 1, + "inactive_split_bytes_current": 10134998016, + "inactive_split_bytes_peak": 12767214080, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:52.565380Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113252524544, + "allocated_bytes_peak_delta": 23623257600, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89629266944, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89629266944, + "device": 1, + "inactive_split_bytes_current": 8870415360, + "inactive_split_bytes_peak": 11623535616, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 94701092864, + "reserved_bytes_peak": 94701092864, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:17:51.446676Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10134998016, + "inactive_split_bytes_current_delta": 1264582656, + "inactive_split_bytes_peak": 12767214080, + "inactive_split_bytes_peak_delta": 1143678464, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 21510488064, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 21510488064, + "segment_count_current": 538, + "segment_count_current_delta": 75, + "segment_count_peak": 538, + "segment_count_peak_delta": 75, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T141752565783Z.pickle", + "timestamp_utc": "2026-08-01T14:17:52.673460Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T141752565783Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T141752565783Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bda3988af2c7c5e56286503bdd6c31649990c3c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T141752565783Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e66382c74822ea8dc2cebc6fe80da793656817cdf74ce00328a38ec4378c9d4 +size 5588584 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T141753784106Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T141753784106Z.json new file mode 100644 index 0000000000000000000000000000000000000000..512ec9a5389a4c857c7b36d3336ed3b01a31766a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T141753784106Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257174528, + "active_bytes_peak_delta": 4649984, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257174528, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257174528, + "device": 1, + "inactive_split_bytes_current": 10464250880, + "inactive_split_bytes_peak": 12767214080, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:53.783719Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257174528, + "allocated_bytes_peak_delta": 4649984, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113252524544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113252524544, + "device": 1, + "inactive_split_bytes_current": 10134998016, + "inactive_split_bytes_peak": 12767214080, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:52.685203Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10464250880, + "inactive_split_bytes_current_delta": 329252864, + "inactive_split_bytes_peak": 12767214080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T141753784106Z.pickle", + "timestamp_utc": "2026-08-01T14:17:53.880460Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T141753784106Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T141753784106Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3c60a6a920c76dee5c8c3b9c6983e81984554953 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T141753784106Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a40cff14a22b8822b82be4dd2300afc373417deed99f3a108eb4a1a4eca72f81 +size 5588498 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T141754984675Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T141754984675Z.json new file mode 100644 index 0000000000000000000000000000000000000000..076bd80a670ec348705d7b3702c909e991900d9c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T141754984675Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257174528, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257174528, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257174528, + "device": 1, + "inactive_split_bytes_current": 10401336320, + "inactive_split_bytes_peak": 12767214080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:54.984259Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257174528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257174528, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257174528, + "device": 1, + "inactive_split_bytes_current": 10464250880, + "inactive_split_bytes_peak": 12767214080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:53.891535Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10401336320, + "inactive_split_bytes_current_delta": -62914560, + "inactive_split_bytes_peak": 12767214080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T141754984675Z.pickle", + "timestamp_utc": "2026-08-01T14:17:55.090762Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T141754984675Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T141754984675Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3bb6b48239282332a29c63a344b4287aeb5294c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T141754984675Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ce409e7ea8ece24d03e7186258fe5370bd500083e75fa0836a2f461df6153dd +size 5597367 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T141756762378Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T141756762378Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd5001e38908a3d46256a5c06377194aa0328f63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T141756762378Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257767936, + "active_bytes_peak_delta": 593408, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257767936, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257767936, + "device": 1, + "inactive_split_bytes_current": 10661383168, + "inactive_split_bytes_peak": 12767214080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:56.761977Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257767936, + "allocated_bytes_peak_delta": 593408, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257174528, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257174528, + "device": 1, + "inactive_split_bytes_current": 10401336320, + "inactive_split_bytes_peak": 12767214080, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:55.104134Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10661383168, + "inactive_split_bytes_current_delta": 260046848, + "inactive_split_bytes_peak": 12767214080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T141756762378Z.pickle", + "timestamp_utc": "2026-08-01T14:17:56.866516Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T141756762378Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T141756762378Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f995faa3a99be58513fa1ca5c9c47819c7c96467 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T141756762378Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dfa1df690cd7053d8193f7ace68374811a6c6bcf3ca28cb81e681a8d0393e7c +size 5596941 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T141757966147Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T141757966147Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5cca4076b6e3f313855d364ec3d5414c51cd7c07 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T141757966147Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113266700800, + "active_bytes_peak_delta": 8932864, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 1, + "inactive_split_bytes_current": 9728150528, + "inactive_split_bytes_peak": 12767214080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:57.965747Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113266700800, + "allocated_bytes_peak_delta": 8932864, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257767936, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257767936, + "device": 1, + "inactive_split_bytes_current": 10661383168, + "inactive_split_bytes_peak": 12767214080, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:56.879562Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9728150528, + "inactive_split_bytes_current_delta": -933232640, + "inactive_split_bytes_peak": 12767214080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T141757966147Z.pickle", + "timestamp_utc": "2026-08-01T14:17:58.065118Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T141757966147Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T141757966147Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2df685d2e5e7890a280e2f30a8e73584a807a2d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T141757966147Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d7b54f45df27bbf4b8e73185e774c266f5de0698cd96f392cef46a97e954e2b +size 5596671 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T141759128798Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T141759128798Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7ba085889b8b579847e1ec2ea889899c72e0584c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T141759128798Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113266700800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 1, + "inactive_split_bytes_current": 10193718272, + "inactive_split_bytes_peak": 12767214080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:59.128406Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113266700800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 1, + "inactive_split_bytes_current": 9728150528, + "inactive_split_bytes_peak": 12767214080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:58.077473Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10193718272, + "inactive_split_bytes_current_delta": 465567744, + "inactive_split_bytes_peak": 12767214080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T141759128798Z.pickle", + "timestamp_utc": "2026-08-01T14:17:59.228803Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T141759128798Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T141759128798Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..933ae7e83106dfa78c4f997f04da8dcfb3df45e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T141759128798Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52d5a1acbe75b64cd538d086d837469a926c20a6d4027a8f59140360ab8cda11 +size 5596899 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T141800313941Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T141800313941Z.json new file mode 100644 index 0000000000000000000000000000000000000000..136b69411a5be1f9aa4879fa29ee028155d1c559 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T141800313941Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113266700800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 1, + "inactive_split_bytes_current": 9610710016, + "inactive_split_bytes_peak": 12767214080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:00.313518Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113266700800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 1, + "inactive_split_bytes_current": 10193718272, + "inactive_split_bytes_peak": 12767214080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:59.241110Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9610710016, + "inactive_split_bytes_current_delta": -583008256, + "inactive_split_bytes_peak": 12767214080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T141800313941Z.pickle", + "timestamp_utc": "2026-08-01T14:18:00.413218Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T141800313941Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T141800313941Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..07e1516c47b50633caaffce430751b95b4054a2c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T141800313941Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b29a7b87ee4953e22a1cbf31dea1a8d9a27b47cb7f0bf91a3f7eaddca4c3e625 +size 5596345 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T141801477690Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T141801477690Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36099cec0d0c6ef9e08301767094208337d98885 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T141801477690Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113266700800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 1, + "inactive_split_bytes_current": 10430696448, + "inactive_split_bytes_peak": 12767214080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:01.477287Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113266700800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 1, + "inactive_split_bytes_current": 9610710016, + "inactive_split_bytes_peak": 12767214080, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:00.425744Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10430696448, + "inactive_split_bytes_current_delta": 819986432, + "inactive_split_bytes_peak": 12767214080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T141801477690Z.pickle", + "timestamp_utc": "2026-08-01T14:18:01.574891Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T141801477690Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T141801477690Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0f2f787728de213c5493e873e1a2a1b225040db0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T141801477690Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6008a164d9cd1617cf05b407778263e9876f3948469a85ca6f939a87faa2cc00 +size 5603450 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T141857701254Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T141857701254Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f766d80950db099bd20f1648e5a62cf8fd427f40 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T141857701254Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113266700800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 113266700800, + "device": 1, + "inactive_split_bytes_current": 9808153600, + "inactive_split_bytes_peak": 12767214080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:57.700788Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113266700800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 1, + "inactive_split_bytes_current": 10430696448, + "inactive_split_bytes_peak": 12767214080, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:01.586965Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9808153600, + "inactive_split_bytes_current_delta": -622542848, + "inactive_split_bytes_peak": 12767214080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T141857701254Z.pickle", + "timestamp_utc": "2026-08-01T14:18:57.893574Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T141857701254Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T141857701254Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6a154a73db1fc899d7c1c3d623d81fea10b0c3d5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T141857701254Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:610572e4e9cc163ef136fb765f1261913a2c22bff17d4ba27108a05ce9432d74 +size 9212435 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T141751322378Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T141751322378Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0a02658af8f94e77716e24704d1be18c9825e698 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T141751322378Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 89629266944, + "active_bytes_peak_delta": 54162930688, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89629266944, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89629266944, + "device": 2, + "inactive_split_bytes_current": 8872512512, + "inactive_split_bytes_peak": 11627729920, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 94701092864, + "reserved_bytes_peak": 94701092864, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:17:51.321813Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 89629266944, + "allocated_bytes_peak_delta": 54162930688, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:16:43.026097Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8872512512, + "inactive_split_bytes_current_delta": 8739693568, + "inactive_split_bytes_peak": 11627729920, + "inactive_split_bytes_peak_delta": 11467343872, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 94701092864, + "reserved_bytes_current_delta": 55457087488, + "reserved_bytes_peak": 94701092864, + "reserved_bytes_peak_delta": 55457087488, + "segment_count_current": 463, + "segment_count_current_delta": 392, + "segment_count_peak": 463, + "segment_count_peak_delta": 392, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T141751322378Z.pickle", + "timestamp_utc": "2026-08-01T14:17:51.432374Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T141751322378Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T141751322378Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..16f5db9959d3865a04a3037f55e59457605225b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T141751322378Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69b80d26ba9f2b726c90d56838b5daa7b0984655414bf4973776f6ed0e5a3f65 +size 5445059 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T141752565699Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T141752565699Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a4a8f328c2576e1c9c076947e3795d5dc9f5dc7b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T141752565699Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257375232, + "active_bytes_peak_delta": 23628108288, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257375232, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257375232, + "device": 2, + "inactive_split_bytes_current": 10642508800, + "inactive_split_bytes_peak": 11652177920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:52.565303Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257375232, + "allocated_bytes_peak_delta": 23628108288, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89629266944, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89629266944, + "device": 2, + "inactive_split_bytes_current": 8872512512, + "inactive_split_bytes_peak": 11627729920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 94701092864, + "reserved_bytes_peak": 94701092864, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:17:51.450243Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10642508800, + "inactive_split_bytes_current_delta": 1769996288, + "inactive_split_bytes_peak": 11652177920, + "inactive_split_bytes_peak_delta": 24448000, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 21510488064, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 21510488064, + "segment_count_current": 538, + "segment_count_current_delta": 75, + "segment_count_peak": 538, + "segment_count_peak_delta": 75, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T141752565699Z.pickle", + "timestamp_utc": "2026-08-01T14:17:52.675937Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T141752565699Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T141752565699Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7bc5454e2be62287541762ff1e25d90cb6ad7540 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T141752565699Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdb226f79be78d0a715fea1b9838c59c1c100d96016c651be8ed3bbe63f688c6 +size 5588425 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T141753784065Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T141753784065Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7df261887fab9ab651e0b054093aca2df3e334fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T141753784065Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257375232, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257375232, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257375232, + "device": 2, + "inactive_split_bytes_current": 10577497088, + "inactive_split_bytes_peak": 11652177920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:53.783676Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257375232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257375232, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257375232, + "device": 2, + "inactive_split_bytes_current": 10642508800, + "inactive_split_bytes_peak": 11652177920, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:52.690453Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10577497088, + "inactive_split_bytes_current_delta": -65011712, + "inactive_split_bytes_peak": 11652177920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T141753784065Z.pickle", + "timestamp_utc": "2026-08-01T14:17:53.887541Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T141753784065Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T141753784065Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..38a915d73188c01c6d654afbdb207530bca15515 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T141753784065Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dc4fc8955597f1eb09cf12a55378352b1e4ab0d32a404aee45377e703a85785 +size 5601466 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T141754980519Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T141754980519Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9dcc81b94906fa59666e3cc376817a523f0ad4f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T141754980519Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261741056, + "active_bytes_peak_delta": 4365824, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 9371634688, + "inactive_split_bytes_peak": 11652177920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:54.980119Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261741056, + "allocated_bytes_peak_delta": 4365824, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257375232, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257375232, + "device": 2, + "inactive_split_bytes_current": 10577497088, + "inactive_split_bytes_peak": 11652177920, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:53.901088Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9371634688, + "inactive_split_bytes_current_delta": -1205862400, + "inactive_split_bytes_peak": 11652177920, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T141754980519Z.pickle", + "timestamp_utc": "2026-08-01T14:17:55.081470Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T141754980519Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T141754980519Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c28515894ff45a04db1806b0417a74f6af0a7665 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T141754980519Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a2ad78337680ed11cc7905460dc66db6a19e222aaf2cd38e2a8b25ff2d58af2 +size 5594712 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T141756762331Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T141756762331Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c939ce01e706e6fdd3d9527c63b22e43668ec7c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T141756762331Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261741056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 11611393024, + "inactive_split_bytes_peak": 12188999680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:56.761920Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261741056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 9371634688, + "inactive_split_bytes_peak": 11652177920, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:55.095183Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 11611393024, + "inactive_split_bytes_current_delta": 2239758336, + "inactive_split_bytes_peak": 12188999680, + "inactive_split_bytes_peak_delta": 536821760, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T141756762331Z.pickle", + "timestamp_utc": "2026-08-01T14:17:56.869433Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T141756762331Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T141756762331Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7daf69392576d50655db0e2035ad27108b7f4218 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T141756762331Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ff21313cc591b84def670aa29f05b717b156a95175b1a7a2930187079dd5c2d +size 5597027 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T141757966094Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T141757966094Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b570edaa7424facfcd98c1b6bab1e7a7acb00e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T141757966094Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261741056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 10317450240, + "inactive_split_bytes_peak": 12188999680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:57.965697Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261741056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 11611393024, + "inactive_split_bytes_peak": 12188999680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:56.882738Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10317450240, + "inactive_split_bytes_current_delta": -1293942784, + "inactive_split_bytes_peak": 12188999680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T141757966094Z.pickle", + "timestamp_utc": "2026-08-01T14:17:58.070377Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T141757966094Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T141757966094Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29d342555b5671a627fe9b8b9fd651a90ceca585 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T141757966094Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14136b27320944cc210ee6d540aed2e9fcb0a9cd171ada86efb6df46946aff7c +size 5596587 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T141759128762Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T141759128762Z.json new file mode 100644 index 0000000000000000000000000000000000000000..623f064c89b91473174ba31f49df29eae3c171f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T141759128762Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261741056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 11231808512, + "inactive_split_bytes_peak": 12188999680, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:59.128356Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261741056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 10317450240, + "inactive_split_bytes_peak": 12188999680, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:58.085605Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 11231808512, + "inactive_split_bytes_current_delta": 914358272, + "inactive_split_bytes_peak": 12188999680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T141759128762Z.pickle", + "timestamp_utc": "2026-08-01T14:17:59.228743Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T141759128762Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T141759128762Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e00d8413fceecc9324efadf0b1a2eb97916301e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T141759128762Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:259adfe0d15e5408e1aa2e5939696d8413ece78bc91dae6aef565f661c37e00d +size 5596826 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T141800313864Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T141800313864Z.json new file mode 100644 index 0000000000000000000000000000000000000000..182cde44feab8fa81f7b67ecfdf00944a6e95030 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T141800313864Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261741056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 10260827136, + "inactive_split_bytes_peak": 12188999680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:00.313457Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261741056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 11231808512, + "inactive_split_bytes_peak": 12188999680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:59.241482Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10260827136, + "inactive_split_bytes_current_delta": -970981376, + "inactive_split_bytes_peak": 12188999680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T141800313864Z.pickle", + "timestamp_utc": "2026-08-01T14:18:00.412124Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T141800313864Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T141800313864Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d62404457e550b71af1cd62903621c5fa0d4716 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T141800313864Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c0585d780b52f456fe7981c21f1dbbf19efeb31b62ffd66361fb3ebbb655681 +size 5596741 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T141801477640Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T141801477640Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e8eceac7ea8ad696e29dd31a70c5e23c9be0dcc4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T141801477640Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261741056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 10139192320, + "inactive_split_bytes_peak": 12188999680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:01.477246Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261741056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 10260827136, + "inactive_split_bytes_peak": 12188999680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:00.424980Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10139192320, + "inactive_split_bytes_current_delta": -121634816, + "inactive_split_bytes_peak": 12188999680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T141801477640Z.pickle", + "timestamp_utc": "2026-08-01T14:18:01.575399Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T141801477640Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T141801477640Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c406f2661cd57e2ce1f2c2ebb389f6b9ad663477 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T141801477640Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94a14899e96ec243abaf7a30abd40d002be8ae34b9ea0661d9b9a103bc42ab3c +size 5606332 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T141857701282Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T141857701282Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c86ec2d6643c47a7a93d1767d56e1ccc5ddbca28 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T141857701282Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113261741056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 9806056448, + "inactive_split_bytes_peak": 12188999680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:57.700813Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113261741056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261741056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261741056, + "device": 2, + "inactive_split_bytes_current": 10139192320, + "inactive_split_bytes_peak": 12188999680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:01.587758Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9806056448, + "inactive_split_bytes_current_delta": -333135872, + "inactive_split_bytes_peak": 12188999680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T141857701282Z.pickle", + "timestamp_utc": "2026-08-01T14:18:57.896251Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T141857701282Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T141857701282Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..853439f6df8837fbca51053d88ed00554c99d929 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T141857701282Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a19eca3f655659750e3f6cd394929f7c3109d7c03927d2b4ad901a6d85e9f046 +size 9212552 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T141751322477Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T141751322477Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d1685a37eb5baa34866f2a442cf7a0db308314c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T141751322477Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 89629266944, + "active_bytes_peak_delta": 54162930688, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89629266944, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89629266944, + "device": 3, + "inactive_split_bytes_current": 8872512512, + "inactive_split_bytes_peak": 11627729920, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 94703190016, + "reserved_bytes_peak": 94703190016, + "segment_count_current": 464, + "segment_count_peak": 464, + "timestamp_utc": "2026-08-01T14:17:51.321883Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 89629266944, + "allocated_bytes_peak_delta": 54162930688, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:16:43.026998Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8872512512, + "inactive_split_bytes_current_delta": 8739693568, + "inactive_split_bytes_peak": 11627729920, + "inactive_split_bytes_peak_delta": 11467343872, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 94703190016, + "reserved_bytes_current_delta": 55459184640, + "reserved_bytes_peak": 94703190016, + "reserved_bytes_peak_delta": 55459184640, + "segment_count_current": 464, + "segment_count_current_delta": 393, + "segment_count_peak": 464, + "segment_count_peak_delta": 393, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T141751322477Z.pickle", + "timestamp_utc": "2026-08-01T14:17:51.435371Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T141751322477Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T141751322477Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9dc392610b0479bddd8fe68e13c8631d5e2203ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T141751322477Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce77e050df9365fea0d345b3474ce391f55d280505cadde59d14d07f0d33bd74 +size 5451471 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T141752565741Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T141752565741Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9e723b3a9eae9e52f54db7b512d1240802ab988 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T141752565741Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113252524544, + "active_bytes_peak_delta": 23623257600, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113252524544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113252524544, + "device": 3, + "inactive_split_bytes_current": 10141289472, + "inactive_split_bytes_peak": 12769311232, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:52.565298Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113252524544, + "allocated_bytes_peak_delta": 23623257600, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89629266944, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89629266944, + "device": 3, + "inactive_split_bytes_current": 8872512512, + "inactive_split_bytes_peak": 11627729920, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 94703190016, + "reserved_bytes_peak": 94703190016, + "segment_count_current": 464, + "segment_count_peak": 464, + "timestamp_utc": "2026-08-01T14:17:51.456468Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10141289472, + "inactive_split_bytes_current_delta": 1268776960, + "inactive_split_bytes_peak": 12769311232, + "inactive_split_bytes_peak_delta": 1141581312, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 21508390912, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 21508390912, + "segment_count_current": 538, + "segment_count_current_delta": 74, + "segment_count_peak": 538, + "segment_count_peak_delta": 74, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T141752565741Z.pickle", + "timestamp_utc": "2026-08-01T14:17:52.680146Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T141752565741Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T141752565741Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fc67a4bfbc9eef12b63b07c5a5c2b17326c52680 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T141752565741Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e480581d2a0b18f2592a15433a76e3bf6af568d6d710dab0806d05e4c4a5f522 +size 5595142 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T141753784088Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T141753784088Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b4e6598d5ec641621ef1015f179a03a08eba3087 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T141753784088Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257174528, + "active_bytes_peak_delta": 4649984, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257174528, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257174528, + "device": 3, + "inactive_split_bytes_current": 10460056576, + "inactive_split_bytes_peak": 12769311232, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:53.783682Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257174528, + "allocated_bytes_peak_delta": 4649984, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113252524544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113252524544, + "device": 3, + "inactive_split_bytes_current": 10141289472, + "inactive_split_bytes_peak": 12769311232, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:52.695212Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10460056576, + "inactive_split_bytes_current_delta": 318767104, + "inactive_split_bytes_peak": 12769311232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T141753784088Z.pickle", + "timestamp_utc": "2026-08-01T14:17:53.890356Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T141753784088Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T141753784088Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..edcd02ef48571316a63d117cc8240fc3c562a67a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T141753784088Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2d3a5fd9777272d33d8eaddb6c73c765d370ee0ade9f0e84b8a9da54aa0fe9e +size 5595292 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T141754980495Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T141754980495Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5555ee2bb7ee01137e0f53edc8a9d85fe49df08 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T141754980495Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257174528, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257174528, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257174528, + "device": 3, + "inactive_split_bytes_current": 10405530624, + "inactive_split_bytes_peak": 12769311232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:54.980078Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257174528, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257174528, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257174528, + "device": 3, + "inactive_split_bytes_current": 10460056576, + "inactive_split_bytes_peak": 12769311232, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:53.905476Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10405530624, + "inactive_split_bytes_current_delta": -54525952, + "inactive_split_bytes_peak": 12769311232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T141754980495Z.pickle", + "timestamp_utc": "2026-08-01T14:17:55.092790Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T141754980495Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T141754980495Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1b8aa4b79cd1adcc4f339d2351aa7c071a96188a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T141754980495Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d40a055a0381c9a4da5240dca8f9ca5a3715f124c3a97f09c7f6eb3742b9655 +size 5604015 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T141756762346Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T141756762346Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a88fdd22712eaf24af0ecc2944dc22ef72eb17f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T141756762346Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257767936, + "active_bytes_peak_delta": 593408, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257767936, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257767936, + "device": 3, + "inactive_split_bytes_current": 10663480320, + "inactive_split_bytes_peak": 12769311232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:56.761920Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257767936, + "allocated_bytes_peak_delta": 593408, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257174528, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257174528, + "device": 3, + "inactive_split_bytes_current": 10405530624, + "inactive_split_bytes_peak": 12769311232, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:55.110317Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10663480320, + "inactive_split_bytes_current_delta": 257949696, + "inactive_split_bytes_peak": 12769311232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T141756762346Z.pickle", + "timestamp_utc": "2026-08-01T14:17:56.869539Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T141756762346Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T141756762346Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a786ecec5a3390adaccc550cac701a8092990591 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T141756762346Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0de2050f9bd843e14074ef90fadc2db4e35e5744ecfc8d439bebfa8d6db3f0f +size 5603976 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T141757966095Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T141757966095Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a4e25ceeddc01e5d50ad18555e1e92985fc72008 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T141757966095Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113266700800, + "active_bytes_peak_delta": 8932864, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 3, + "inactive_split_bytes_current": 9736539136, + "inactive_split_bytes_peak": 12769311232, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:57.965684Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113266700800, + "allocated_bytes_peak_delta": 8932864, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257767936, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257767936, + "device": 3, + "inactive_split_bytes_current": 10663480320, + "inactive_split_bytes_peak": 12769311232, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:56.884902Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9736539136, + "inactive_split_bytes_current_delta": -926941184, + "inactive_split_bytes_peak": 12769311232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T141757966095Z.pickle", + "timestamp_utc": "2026-08-01T14:17:58.070286Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T141757966095Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T141757966095Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ca66bf18c4bbc3e0a4c7216788aa5fef023b400 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T141757966095Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7604c4d5f9b335ca022414d9178ff7ee400abcf27f0fc319a0853017d19cf8d +size 5606250 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T141759128738Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T141759128738Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c9e90c27a683a96462de875d16683322b46365d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T141759128738Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113266700800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 3, + "inactive_split_bytes_current": 10200009728, + "inactive_split_bytes_peak": 12769311232, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:59.128326Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113266700800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 3, + "inactive_split_bytes_current": 9736539136, + "inactive_split_bytes_peak": 12769311232, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:58.085180Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10200009728, + "inactive_split_bytes_current_delta": 463470592, + "inactive_split_bytes_peak": 12769311232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T141759128738Z.pickle", + "timestamp_utc": "2026-08-01T14:17:59.232132Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T141759128738Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T141759128738Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f3fefa74c9147ab57c0129667eb6092df37ec09 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T141759128738Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d3b2265bec45c4708b750d43e1ed9ec8dd6c72e857f84836ec516976a3cb992 +size 5603405 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T141800313927Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T141800313927Z.json new file mode 100644 index 0000000000000000000000000000000000000000..555255aebf6549c44a9d9fb9d4df002029f92d70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T141800313927Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113266700800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 3, + "inactive_split_bytes_current": 9608612864, + "inactive_split_bytes_peak": 12769311232, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:00.313486Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113266700800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 3, + "inactive_split_bytes_current": 10200009728, + "inactive_split_bytes_peak": 12769311232, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:59.247165Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9608612864, + "inactive_split_bytes_current_delta": -591396864, + "inactive_split_bytes_peak": 12769311232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T141800313927Z.pickle", + "timestamp_utc": "2026-08-01T14:18:00.416786Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T141800313927Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T141800313927Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de350f5fc7f84430e01b68bdb47c458a32bea196 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T141800313927Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1f7315b594f8545aa82a19529a60dfde7d9f52cd3ee48eba4abc117d91e4072 +size 5603451 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T141801477662Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T141801477662Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04e0445a218189f8d2ac2e35dfab7def518fe690 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T141801477662Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113266700800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 3, + "inactive_split_bytes_current": 10581691392, + "inactive_split_bytes_peak": 12769311232, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:01.477249Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113266700800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 3, + "inactive_split_bytes_current": 9608612864, + "inactive_split_bytes_peak": 12769311232, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:00.431323Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10581691392, + "inactive_split_bytes_current_delta": 973078528, + "inactive_split_bytes_peak": 12769311232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T141801477662Z.pickle", + "timestamp_utc": "2026-08-01T14:18:01.579220Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T141801477662Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T141801477662Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..23dbf56f3525a402025fe7b5ea3b0b0ca31f7998 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T141801477662Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a9d134580a641fe9d8639a06ffdb97b12dcc8d3c0a214807a0bd2b3a6288bbb +size 5603683 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T141857701269Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T141857701269Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d0e5422c8802ad077333123253d55823b553daca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T141857701269Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113266700800, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 113266700800, + "device": 3, + "inactive_split_bytes_current": 9749433344, + "inactive_split_bytes_peak": 12769311232, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:57.700805Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113266700800, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113266700800, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113266700800, + "device": 3, + "inactive_split_bytes_current": 10581691392, + "inactive_split_bytes_peak": 12769311232, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:01.593946Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9749433344, + "inactive_split_bytes_current_delta": -832258048, + "inactive_split_bytes_peak": 12769311232, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T141857701269Z.pickle", + "timestamp_utc": "2026-08-01T14:18:57.904630Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T141857701269Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T141857701269Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3d51d31a2d681aad6fadba553dc99ce03faf8326 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T141857701269Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:547645aba691ee4b2781fcc9d69fac464fdaf47dc0f915e53b85e20cefd1fcc2 +size 9212750 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T141751322369Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T141751322369Z.json new file mode 100644 index 0000000000000000000000000000000000000000..772293d1c1bfd2ff0ee4a25efccc3e5225252013 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T141751322369Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855005184, + "active_bytes_current_delta": 31623107584, + "active_bytes_peak": 89628873216, + "active_bytes_peak_delta": 54162094592, + "after": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 89628873216, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 89628873216, + "device": 4, + "inactive_split_bytes_current": 9709917184, + "inactive_split_bytes_peak": 11272249344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 95976161280, + "reserved_bytes_peak": 95976161280, + "segment_count_current": 469, + "segment_count_peak": 469, + "timestamp_utc": "2026-08-01T14:17:51.321955Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855005184, + "allocated_bytes_current_delta": 31623107584, + "allocated_bytes_peak": 89628873216, + "allocated_bytes_peak_delta": 54162094592, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:16:43.025343Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9709917184, + "inactive_split_bytes_current_delta": 9577540608, + "inactive_split_bytes_peak": 11272249344, + "inactive_split_bytes_peak_delta": 11118839808, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 95976161280, + "reserved_bytes_current_delta": 56732155904, + "reserved_bytes_peak": 95976161280, + "reserved_bytes_peak_delta": 56732155904, + "segment_count_current": 469, + "segment_count_current_delta": 398, + "segment_count_peak": 469, + "segment_count_peak_delta": 398, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T141751322369Z.pickle", + "timestamp_utc": "2026-08-01T14:17:51.431741Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T141751322369Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T141751322369Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76bdb2974a7b71a6d5893e785a059a8cfde798e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T141751322369Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d40c42f443a76f1a2f2fe1b1d413757c549b87bfc13155591d171d6093b5316c +size 5377509 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T141752565642Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T141752565642Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1e7670e3f626c420017bf0f5cf2fc0de58a5aaab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T141752565642Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855005184, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113255977472, + "active_bytes_peak_delta": 23627104256, + "after": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113255977472, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113255977472, + "device": 4, + "inactive_split_bytes_current": 11574285312, + "inactive_split_bytes_peak": 12291356672, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:17:52.565248Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855005184, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113255977472, + "allocated_bytes_peak_delta": 23627104256, + "before": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 89628873216, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 89628873216, + "device": 4, + "inactive_split_bytes_current": 9709917184, + "inactive_split_bytes_peak": 11272249344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 95976161280, + "reserved_bytes_peak": 95976161280, + "segment_count_current": 469, + "segment_count_peak": 469, + "timestamp_utc": "2026-08-01T14:17:51.447939Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11574285312, + "inactive_split_bytes_current_delta": 1864368128, + "inactive_split_bytes_peak": 12291356672, + "inactive_split_bytes_peak_delta": 1019107328, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115639058432, + "reserved_bytes_current_delta": 19662897152, + "reserved_bytes_peak": 115639058432, + "reserved_bytes_peak_delta": 19662897152, + "segment_count_current": 530, + "segment_count_current_delta": 61, + "segment_count_peak": 530, + "segment_count_peak_delta": 61, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T141752565642Z.pickle", + "timestamp_utc": "2026-08-01T14:17:52.674346Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T141752565642Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T141752565642Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7a9486b83c80fa09e813c7fdae6585c5c89bcd0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T141752565642Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b19b9fa0f442f5630822bb591b1da9557301a442676ce35e0fe94bcaffb11a2c +size 5602002 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T141753784024Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T141753784024Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f6f8613a028d4f022bee76f46d8322c8fbda7db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T141753784024Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855005184, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113255977472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113255977472, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113255977472, + "device": 4, + "inactive_split_bytes_current": 11630908416, + "inactive_split_bytes_peak": 12892338176, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:17:53.783636Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855005184, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113255977472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113255977472, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113255977472, + "device": 4, + "inactive_split_bytes_current": 11574285312, + "inactive_split_bytes_peak": 12291356672, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:17:52.688260Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11630908416, + "inactive_split_bytes_current_delta": 56623104, + "inactive_split_bytes_peak": 12892338176, + "inactive_split_bytes_peak_delta": 600981504, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115639058432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115639058432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T141753784024Z.pickle", + "timestamp_utc": "2026-08-01T14:17:53.880836Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T141753784024Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T141753784024Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..faa2bb796aa547e5aafc32dbda9da970691177f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T141753784024Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb30acc0a3044cc58be578bfd8cb745c32e5f492a44875e6b445d4e6e6593eaa +size 5608962 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T141754980459Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T141754980459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7a7b8edace38f60e3a93aa5c9f735b48c20b29fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T141754980459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855005184, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257091584, + "active_bytes_peak_delta": 1114112, + "after": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113257091584, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113257091584, + "device": 4, + "inactive_split_bytes_current": 10655732736, + "inactive_split_bytes_peak": 12892338176, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:17:54.980056Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855005184, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257091584, + "allocated_bytes_peak_delta": 1114112, + "before": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113255977472, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113255977472, + "device": 4, + "inactive_split_bytes_current": 11630908416, + "inactive_split_bytes_peak": 12892338176, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:17:53.892790Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10655732736, + "inactive_split_bytes_current_delta": -975175680, + "inactive_split_bytes_peak": 12892338176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115639058432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115639058432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T141754980459Z.pickle", + "timestamp_utc": "2026-08-01T14:17:55.080015Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T141754980459Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T141754980459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03972f98137a7e6bca19ca3ad8f4896fa5d17762 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T141754980459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ba4082938c62a5811db2f47fedfd74ecada54edf41fd64dc95c22b44e281074 +size 5611236 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T141756762293Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T141756762293Z.json new file mode 100644 index 0000000000000000000000000000000000000000..639a469fb33d4eea069e00cb019163c49fa1201b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T141756762293Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855005184, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259582976, + "active_bytes_peak_delta": 2491392, + "after": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113259582976, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113259582976, + "device": 4, + "inactive_split_bytes_current": 11419096064, + "inactive_split_bytes_peak": 12892338176, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:17:56.761888Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855005184, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259582976, + "allocated_bytes_peak_delta": 2491392, + "before": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113257091584, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113257091584, + "device": 4, + "inactive_split_bytes_current": 10655732736, + "inactive_split_bytes_peak": 12892338176, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:17:55.091715Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11419096064, + "inactive_split_bytes_current_delta": 763363328, + "inactive_split_bytes_peak": 12892338176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115639058432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115639058432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T141756762293Z.pickle", + "timestamp_utc": "2026-08-01T14:17:56.863528Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T141756762293Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T141756762293Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..840c998f3401e49978e4e2e36aee2f6529784872 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T141756762293Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a8aede45cf39a5bf9d0a15cd0a62e4e28b178739666aa388af9a74600f5c7a0 +size 5610919 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T141757966040Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T141757966040Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d1a324be70edfbdf63336d71383afa333cf74c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T141757966040Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855005184, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259582976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113259582976, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113259582976, + "device": 4, + "inactive_split_bytes_current": 11454747648, + "inactive_split_bytes_peak": 12892338176, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:17:57.965654Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855005184, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259582976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113259582976, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113259582976, + "device": 4, + "inactive_split_bytes_current": 11419096064, + "inactive_split_bytes_peak": 12892338176, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:17:56.875330Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11454747648, + "inactive_split_bytes_current_delta": 35651584, + "inactive_split_bytes_peak": 12892338176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115639058432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115639058432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T141757966040Z.pickle", + "timestamp_utc": "2026-08-01T14:17:58.058963Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T141757966040Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T141757966040Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b27ad71b77fe1a9f3c9e22705cbb76279fc0b3a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T141757966040Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43f09e21ecdadd40ad43ae761bcfa3fc7f12e128dd8d265ba88adea430e14628 +size 5608019 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T141759128741Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T141759128741Z.json new file mode 100644 index 0000000000000000000000000000000000000000..25245031c907125da85faec22754d90fa5f91568 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T141759128741Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855005184, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259582976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113259582976, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113259582976, + "device": 4, + "inactive_split_bytes_current": 11922412544, + "inactive_split_bytes_peak": 12892338176, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:17:59.128324Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855005184, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259582976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113259582976, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113259582976, + "device": 4, + "inactive_split_bytes_current": 11454747648, + "inactive_split_bytes_peak": 12892338176, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:17:58.069951Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11922412544, + "inactive_split_bytes_current_delta": 467664896, + "inactive_split_bytes_peak": 12892338176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115639058432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115639058432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T141759128741Z.pickle", + "timestamp_utc": "2026-08-01T14:17:59.226285Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T141759128741Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T141759128741Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d69e0f3ffc5fa1dc0abc8eaddf86b703278d06b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T141759128741Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a2dc13dd9bdf52c9accfc3edda29de95cf6044c7a78fbbc83b34ef4f88d705f +size 5607824 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T141800313891Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T141800313891Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36735fd2bcb220f49c25da23265a96e55079334b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T141800313891Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855005184, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259582976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113259582976, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113259582976, + "device": 4, + "inactive_split_bytes_current": 11658171392, + "inactive_split_bytes_peak": 12892338176, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:18:00.313462Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855005184, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259582976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113259582976, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113259582976, + "device": 4, + "inactive_split_bytes_current": 11922412544, + "inactive_split_bytes_peak": 12892338176, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:17:59.237814Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11658171392, + "inactive_split_bytes_current_delta": -264241152, + "inactive_split_bytes_peak": 12892338176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115639058432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115639058432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T141800313891Z.pickle", + "timestamp_utc": "2026-08-01T14:18:00.409989Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T141800313891Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T141800313891Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b3f266157e9af0f67ac257adfe13d2f60ebde761 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T141800313891Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5cb94984230d9cd2cb41a0163424a51ae8c741463a6e4b4264079ab2e7a560a +size 5607928 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T141801477600Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T141801477600Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ec64e73f78ec4136e68716c59e5c499d87f26a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T141801477600Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855005184, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259582976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113259582976, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113259582976, + "device": 4, + "inactive_split_bytes_current": 11098231808, + "inactive_split_bytes_peak": 12892338176, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:18:01.477206Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855005184, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259582976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113259582976, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113259582976, + "device": 4, + "inactive_split_bytes_current": 11658171392, + "inactive_split_bytes_peak": 12892338176, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:18:00.422315Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11098231808, + "inactive_split_bytes_current_delta": -559939584, + "inactive_split_bytes_peak": 12892338176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115639058432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115639058432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T141801477600Z.pickle", + "timestamp_utc": "2026-08-01T14:18:01.571174Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T141801477600Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T141801477600Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..80ccccc0e99fbd5d2663e4f4cee5381c782141c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T141801477600Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1999dec3eae3f5bbd07451f1f2b47320efec1e00dd631e1a3c27e9f745d39c79 +size 5610410 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T141857701351Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T141857701351Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c88a860ead984947a2562425218b33b44ebf84dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T141857701351Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804362240, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113259582976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66804362240, + "active_bytes_peak": 113259582976, + "allocated_bytes_current": 66804362240, + "allocated_bytes_peak": 113259582976, + "device": 4, + "inactive_split_bytes_current": 10979005440, + "inactive_split_bytes_peak": 12892338176, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:18:57.700800Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804362240, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113259582976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855005184, + "active_bytes_peak": 113259582976, + "allocated_bytes_current": 66855005184, + "allocated_bytes_peak": 113259582976, + "device": 4, + "inactive_split_bytes_current": 11098231808, + "inactive_split_bytes_peak": 12892338176, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 115639058432, + "reserved_bytes_peak": 115639058432, + "segment_count_current": 530, + "segment_count_peak": 530, + "timestamp_utc": "2026-08-01T14:18:01.581867Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10979005440, + "inactive_split_bytes_current_delta": -119226368, + "inactive_split_bytes_peak": 12892338176, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 115639058432, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 115639058432, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 530, + "segment_count_current_delta": 0, + "segment_count_peak": 530, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T141857701351Z.pickle", + "timestamp_utc": "2026-08-01T14:18:57.901642Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T141857701351Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T141857701351Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0c49638cbc33585e50328dc36e542498d9f657da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T141857701351Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3596a48ecb6de7cd8e11f02b4a22c7f1f2dcb3ce16914edef5cc1359e24a8a28 +size 9215100 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T141751322428Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T141751322428Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61a845dfd65c16d9495be8e3b484686e3f091039 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T141751322428Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 89629266944, + "active_bytes_peak_delta": 54162930688, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89629266944, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89629266944, + "device": 5, + "inactive_split_bytes_current": 8872512512, + "inactive_split_bytes_peak": 11609830400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 94701092864, + "reserved_bytes_peak": 94701092864, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:17:51.321834Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 89629266944, + "allocated_bytes_peak_delta": 54162930688, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:16:43.026434Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8872512512, + "inactive_split_bytes_current_delta": 8739693568, + "inactive_split_bytes_peak": 11609830400, + "inactive_split_bytes_peak_delta": 11449444352, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 94701092864, + "reserved_bytes_current_delta": 55457087488, + "reserved_bytes_peak": 94701092864, + "reserved_bytes_peak_delta": 55457087488, + "segment_count_current": 463, + "segment_count_current_delta": 392, + "segment_count_peak": 463, + "segment_count_peak_delta": 392, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T141751322428Z.pickle", + "timestamp_utc": "2026-08-01T14:17:51.433490Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T141751322428Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T141751322428Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2058115b414ae37245cbf85dccbaab66c7595d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T141751322428Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e986509511345f8b4e765e6433943429842f79ef6ad2c428a5c7c78ac4c63ff5 +size 5445020 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T141752565698Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T141752565698Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cadb49239846db5655847f4998fdb43c6f4abce9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T141752565698Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257346560, + "active_bytes_peak_delta": 23628079616, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 5, + "inactive_split_bytes_current": 11204545536, + "inactive_split_bytes_peak": 11976654848, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:52.565300Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257346560, + "allocated_bytes_peak_delta": 23628079616, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89629266944, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89629266944, + "device": 5, + "inactive_split_bytes_current": 8872512512, + "inactive_split_bytes_peak": 11609830400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 94701092864, + "reserved_bytes_peak": 94701092864, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:17:51.454593Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11204545536, + "inactive_split_bytes_current_delta": 2332033024, + "inactive_split_bytes_peak": 11976654848, + "inactive_split_bytes_peak_delta": 366824448, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 21510488064, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 21510488064, + "segment_count_current": 538, + "segment_count_current_delta": 75, + "segment_count_peak": 538, + "segment_count_peak_delta": 75, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T141752565698Z.pickle", + "timestamp_utc": "2026-08-01T14:17:52.677331Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T141752565698Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T141752565698Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f99cf6f4bf80c53b1e9ab600fde52e66407371c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T141752565698Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27d21f0dfde21ca4d397dddaeb16c4d54b1037f250037a865931eb60fa59929e +size 5588336 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T141753784076Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T141753784076Z.json new file mode 100644 index 0000000000000000000000000000000000000000..708b97c9e1bd6ae1c64f59c21ae3087f58f9e88b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T141753784076Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257346560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 5, + "inactive_split_bytes_current": 10848029696, + "inactive_split_bytes_peak": 11976654848, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:53.783671Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257346560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 5, + "inactive_split_bytes_current": 11204545536, + "inactive_split_bytes_peak": 11976654848, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:52.692328Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10848029696, + "inactive_split_bytes_current_delta": -356515840, + "inactive_split_bytes_peak": 11976654848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T141753784076Z.pickle", + "timestamp_utc": "2026-08-01T14:17:53.891047Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T141753784076Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T141753784076Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f811f40c065d5cebae7874d711ffb14745c3255e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T141753784076Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fae16d107e6256fcdb595d10df90c10197e57fe787dddc876b397292c25e2d0 +size 5597288 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T141754980511Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T141754980511Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0e908acbfce44c3da18472c47d22221e9e3f9c2d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T141754980511Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257346560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 5, + "inactive_split_bytes_current": 10200009728, + "inactive_split_bytes_peak": 11976654848, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:54.980107Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257346560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 5, + "inactive_split_bytes_current": 10848029696, + "inactive_split_bytes_peak": 11976654848, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:53.906530Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10200009728, + "inactive_split_bytes_current_delta": -648019968, + "inactive_split_bytes_peak": 11976654848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T141754980511Z.pickle", + "timestamp_utc": "2026-08-01T14:17:55.083893Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T141754980511Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T141754980511Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4ebd2926ac0696cf9516c8d9d0f79dce3cac32e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T141754980511Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d33ae0d5248d23575a97e6e7788a6a64470fff86f0730723479c37a2aff8253 +size 5597661 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T141756762346Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T141756762346Z.json new file mode 100644 index 0000000000000000000000000000000000000000..729d0971129f1ebc506a271a4783dda5c1a551e1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T141756762346Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257346560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 5, + "inactive_split_bytes_current": 9872854016, + "inactive_split_bytes_peak": 11976654848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:56.761931Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257346560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 5, + "inactive_split_bytes_current": 10200009728, + "inactive_split_bytes_peak": 11976654848, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:55.098441Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9872854016, + "inactive_split_bytes_current_delta": -327155712, + "inactive_split_bytes_peak": 11976654848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T141756762346Z.pickle", + "timestamp_utc": "2026-08-01T14:17:56.871854Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T141756762346Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T141756762346Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8a064767b903df1bbfaa031c829122afeb965d91 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T141756762346Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05a9ab83d8de80c423121904270af48ff91398e7733c5278d8b4328f72444988 +size 5593894 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T141757966099Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T141757966099Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4aa142eca6764f111d04b4470b1a13cb66f05905 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T141757966099Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260258304, + "active_bytes_peak_delta": 2911744, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260258304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260258304, + "device": 5, + "inactive_split_bytes_current": 10080472064, + "inactive_split_bytes_peak": 11976654848, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:57.965699Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260258304, + "allocated_bytes_peak_delta": 2911744, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 5, + "inactive_split_bytes_current": 9872854016, + "inactive_split_bytes_peak": 11976654848, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:56.889587Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10080472064, + "inactive_split_bytes_current_delta": 207618048, + "inactive_split_bytes_peak": 11976654848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T141757966099Z.pickle", + "timestamp_utc": "2026-08-01T14:17:58.067754Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T141757966099Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T141757966099Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7fa8679dd250af17ada2af7aafd1a8ff74bddd1c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T141757966099Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab76a504a26aea04c859a96e235e86c6e63f61f3ce9396fbff7ca50e33d189a8 +size 5594565 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T141759128763Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T141759128763Z.json new file mode 100644 index 0000000000000000000000000000000000000000..56f584cf0b0dd84e09f3f425135e78b53200843e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T141759128763Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260258304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260258304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260258304, + "device": 5, + "inactive_split_bytes_current": 9906408448, + "inactive_split_bytes_peak": 11976654848, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:59.128359Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260258304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260258304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260258304, + "device": 5, + "inactive_split_bytes_current": 10080472064, + "inactive_split_bytes_peak": 11976654848, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:58.082341Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9906408448, + "inactive_split_bytes_current_delta": -174063616, + "inactive_split_bytes_peak": 11976654848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T141759128763Z.pickle", + "timestamp_utc": "2026-08-01T14:17:59.231075Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T141759128763Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T141759128763Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..faf90e296960dfa1a6d11aa66f9ed94bb0f8afc6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T141759128763Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86f91aee1b88c2d4f42bb924d81939154df780f9fd0fa7106a1bfc3a6110503b +size 5594089 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T141800313870Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T141800313870Z.json new file mode 100644 index 0000000000000000000000000000000000000000..901b387fa9e1523c426edcc2175e72b452033110 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T141800313870Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260258304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260258304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260258304, + "device": 5, + "inactive_split_bytes_current": 10220981248, + "inactive_split_bytes_peak": 11976654848, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:00.313456Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260258304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260258304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260258304, + "device": 5, + "inactive_split_bytes_current": 9906408448, + "inactive_split_bytes_peak": 11976654848, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:59.245915Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10220981248, + "inactive_split_bytes_current_delta": 314572800, + "inactive_split_bytes_peak": 11976654848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T141800313870Z.pickle", + "timestamp_utc": "2026-08-01T14:18:00.416686Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T141800313870Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T141800313870Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..345c8e543c01573d977c376fc153bf22f26776e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T141800313870Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cee34ef9d6ab7f6966e0b7b3d96db4984f17d2c7bcd95afc4550bda722f2e71 +size 5596954 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T141801477638Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T141801477638Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c2b8ee2c0026fa7780f084f9e3fab3ab221c3fb0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T141801477638Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113262024192, + "active_bytes_peak_delta": 1765888, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262024192, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262024192, + "device": 5, + "inactive_split_bytes_current": 11143728128, + "inactive_split_bytes_peak": 11976654848, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:01.477240Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113262024192, + "allocated_bytes_peak_delta": 1765888, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260258304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260258304, + "device": 5, + "inactive_split_bytes_current": 10220981248, + "inactive_split_bytes_peak": 11976654848, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:00.431075Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 11143728128, + "inactive_split_bytes_current_delta": 922746880, + "inactive_split_bytes_peak": 11976654848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T141801477638Z.pickle", + "timestamp_utc": "2026-08-01T14:18:01.578055Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T141801477638Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T141801477638Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f0063c4ac8f808668162e3541e123a0dbc2ea370 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T141801477638Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf3c965ad89473a6dcdff40978e3d23e6d73b40cfd38482372a3d4362e47b262 +size 5593961 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T141857701273Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T141857701273Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b79a3d4f7cd9e18397903f81dc262b98dddc57b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T141857701273Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113262024192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 113262024192, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 113262024192, + "device": 5, + "inactive_split_bytes_current": 9808153600, + "inactive_split_bytes_peak": 11976654848, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:57.700807Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113262024192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262024192, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262024192, + "device": 5, + "inactive_split_bytes_current": 11143728128, + "inactive_split_bytes_peak": 11976654848, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:01.592589Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9808153600, + "inactive_split_bytes_current_delta": -1335574528, + "inactive_split_bytes_peak": 11976654848, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T141857701273Z.pickle", + "timestamp_utc": "2026-08-01T14:18:57.893860Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T141857701273Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T141857701273Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2cff0f62bc4924fed1736e49df2c8c92a8e1022e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T141857701273Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:374290b99366dfdae9be6d3b4c3ad5b5b2aec8f2f8a298bc93142b8f6869519a +size 9212659 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T141751322408Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T141751322408Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b8c5d9c9111cbd60ad063f73fa7767bf8ff76b7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T141751322408Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 89629266944, + "active_bytes_peak_delta": 54162930688, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89629266944, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89629266944, + "device": 6, + "inactive_split_bytes_current": 8870415360, + "inactive_split_bytes_peak": 11605636096, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 94701092864, + "reserved_bytes_peak": 94701092864, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:17:51.321811Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 89629266944, + "allocated_bytes_peak_delta": 54162930688, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:16:43.026011Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8870415360, + "inactive_split_bytes_current_delta": 8737596416, + "inactive_split_bytes_peak": 11605636096, + "inactive_split_bytes_peak_delta": 11445250048, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 94701092864, + "reserved_bytes_current_delta": 55457087488, + "reserved_bytes_peak": 94701092864, + "reserved_bytes_peak_delta": 55457087488, + "segment_count_current": 463, + "segment_count_current_delta": 392, + "segment_count_peak": 463, + "segment_count_peak_delta": 392, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T141751322408Z.pickle", + "timestamp_utc": "2026-08-01T14:17:51.432283Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T141751322408Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T141751322408Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..91d0046f600e39cb32bf457a41c88305f4683022 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T141751322408Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc87d854c993dc198142c92992620e0d459b7d01c567433349a159d0b0d8b190 +size 5444759 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T141752565705Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T141752565705Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f0f353d7dfd2ff4d4602617f449fda7e0d147e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T141752565705Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257346560, + "active_bytes_peak_delta": 23628079616, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 6, + "inactive_split_bytes_current": 11196156928, + "inactive_split_bytes_peak": 11968266240, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:52.565302Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257346560, + "allocated_bytes_peak_delta": 23628079616, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89629266944, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89629266944, + "device": 6, + "inactive_split_bytes_current": 8870415360, + "inactive_split_bytes_peak": 11605636096, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 94701092864, + "reserved_bytes_peak": 94701092864, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:17:51.450148Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 11196156928, + "inactive_split_bytes_current_delta": 2325741568, + "inactive_split_bytes_peak": 11968266240, + "inactive_split_bytes_peak_delta": 362630144, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 21510488064, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 21510488064, + "segment_count_current": 538, + "segment_count_current_delta": 75, + "segment_count_peak": 538, + "segment_count_peak_delta": 75, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T141752565705Z.pickle", + "timestamp_utc": "2026-08-01T14:17:52.676123Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T141752565705Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T141752565705Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d046d322cb6c144aec7740f95cb894d298def57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T141752565705Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54ee5fab792745d2dba9f3e95d5c20055b9f34b18932f9b66dd8ee6f95e351bd +size 5588089 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T141753784047Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T141753784047Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a74ec8b015cff3baf4d7349c8025baca78ebd4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T141753784047Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257346560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 6, + "inactive_split_bytes_current": 10852224000, + "inactive_split_bytes_peak": 11968266240, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:53.783657Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257346560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 6, + "inactive_split_bytes_current": 11196156928, + "inactive_split_bytes_peak": 11968266240, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:52.690659Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10852224000, + "inactive_split_bytes_current_delta": -343932928, + "inactive_split_bytes_peak": 11968266240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T141753784047Z.pickle", + "timestamp_utc": "2026-08-01T14:17:53.886323Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T141753784047Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T141753784047Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2f6f6340914af50965df96045f03da85c3e15d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T141753784047Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d22f71810cef1acc5727b550f6c804440ce2064c2989f9a883334ef827fd5946 +size 5594815 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T141754980556Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T141754980556Z.json new file mode 100644 index 0000000000000000000000000000000000000000..88cc1b2dfb245a517e4f1fcaa60e1fbaaf5d3560 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T141754980556Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257346560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 6, + "inactive_split_bytes_current": 10204204032, + "inactive_split_bytes_peak": 11968266240, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:54.980142Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257346560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 6, + "inactive_split_bytes_current": 10852224000, + "inactive_split_bytes_peak": 11968266240, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:53.899801Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10204204032, + "inactive_split_bytes_current_delta": -648019968, + "inactive_split_bytes_peak": 11968266240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T141754980556Z.pickle", + "timestamp_utc": "2026-08-01T14:17:55.090714Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T141754980556Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T141754980556Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..570fe3db3bcd6d1e4522f2f2850a222bc539652c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T141754980556Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68ff85c8766568e39a89d25d45b4e58c5b84399bf050e19aded3bc00b60e5f72 +size 5594648 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T141756762326Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T141756762326Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9247594a69ba77c14b09389cebfbd41dc7007f32 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T141756762326Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257346560, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 6, + "inactive_split_bytes_current": 9872854016, + "inactive_split_bytes_peak": 11968266240, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:56.761915Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257346560, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 6, + "inactive_split_bytes_current": 10204204032, + "inactive_split_bytes_peak": 11968266240, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:55.107328Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9872854016, + "inactive_split_bytes_current_delta": -331350016, + "inactive_split_bytes_peak": 11968266240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T141756762326Z.pickle", + "timestamp_utc": "2026-08-01T14:17:56.867107Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T141756762326Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T141756762326Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6094a3d9c67baa6a112eee4234955280b09c6eb0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T141756762326Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:332c02af0e68501b902df89476259d4ec654280a175ea21b8af52569eec3becb +size 5603489 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T141757966089Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T141757966089Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15868f557fdff79d0fd50fcf10f9ec8d6638c021 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T141757966089Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260258304, + "active_bytes_peak_delta": 2911744, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260258304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260258304, + "device": 6, + "inactive_split_bytes_current": 10084666368, + "inactive_split_bytes_peak": 11968266240, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:57.965674Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260258304, + "allocated_bytes_peak_delta": 2911744, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113257346560, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113257346560, + "device": 6, + "inactive_split_bytes_current": 9872854016, + "inactive_split_bytes_peak": 11968266240, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:56.880282Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10084666368, + "inactive_split_bytes_current_delta": 211812352, + "inactive_split_bytes_peak": 11968266240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T141757966089Z.pickle", + "timestamp_utc": "2026-08-01T14:17:58.064073Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T141757966089Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T141757966089Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d07329b9d12552cd2e9a43854775f04a39740017 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T141757966089Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d24c1eeff88c26169df93ab86731954af7ef358af7fb566b1701936aff8061a +size 5594250 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T141759128819Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T141759128819Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ba908f9eeb6e7574617867bef0051fc334e915a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T141759128819Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260258304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260258304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260258304, + "device": 6, + "inactive_split_bytes_current": 9910602752, + "inactive_split_bytes_peak": 11968266240, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:59.128330Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260258304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260258304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260258304, + "device": 6, + "inactive_split_bytes_current": 10084666368, + "inactive_split_bytes_peak": 11968266240, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:58.076639Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9910602752, + "inactive_split_bytes_current_delta": -174063616, + "inactive_split_bytes_peak": 11968266240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T141759128819Z.pickle", + "timestamp_utc": "2026-08-01T14:17:59.229961Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T141759128819Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T141759128819Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5068cdcd8ea4ec4591ef0c4f82df4a16768a216 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T141759128819Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b89f0c3eb33936f190e6d3f46049579c280d4af3c82fb96dc07592a23c8a09cd +size 5594326 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T141800313864Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T141800313864Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eb1ebb2fe77b8c8aa9c1ada1acb19f9d58a09108 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T141800313864Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260258304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260258304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260258304, + "device": 6, + "inactive_split_bytes_current": 10220981248, + "inactive_split_bytes_peak": 11968266240, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:00.313448Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260258304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260258304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260258304, + "device": 6, + "inactive_split_bytes_current": 9910602752, + "inactive_split_bytes_peak": 11968266240, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:59.242949Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10220981248, + "inactive_split_bytes_current_delta": 310378496, + "inactive_split_bytes_peak": 11968266240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T141800313864Z.pickle", + "timestamp_utc": "2026-08-01T14:18:00.419018Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T141800313864Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T141800313864Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..91c5b9c5e8fad9a5f7e71feee987c05f402b79b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T141800313864Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f250e23522d0dcfbcbd9e1fd1aef2383ac2ef2bcbe83b18a7aea5ef784f4e10d +size 5593936 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T141801477647Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T141801477647Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c6223b24fe4b173b9197cb00e55609cfefdeac5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T141801477647Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113262024192, + "active_bytes_peak_delta": 1765888, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262024192, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262024192, + "device": 6, + "inactive_split_bytes_current": 11141630976, + "inactive_split_bytes_peak": 11968266240, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:01.477239Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113262024192, + "allocated_bytes_peak_delta": 1765888, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260258304, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260258304, + "device": 6, + "inactive_split_bytes_current": 10220981248, + "inactive_split_bytes_peak": 11968266240, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:00.432339Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 11141630976, + "inactive_split_bytes_current_delta": 920649728, + "inactive_split_bytes_peak": 11968266240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T141801477647Z.pickle", + "timestamp_utc": "2026-08-01T14:18:01.576316Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T141801477647Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T141801477647Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ec10b9c5a4b47f43fc1bd0d86773dee4f65958b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T141801477647Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f4d8ca55a0b07056fe99877edcc9d36a398a63444535da8f592710007b0660 +size 5593970 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T141857701269Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T141857701269Z.json new file mode 100644 index 0000000000000000000000000000000000000000..267aeab03dcc003a5a9d4f4345e9626778963848 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T141857701269Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113262024192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 113262024192, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 113262024192, + "device": 6, + "inactive_split_bytes_current": 9808153600, + "inactive_split_bytes_peak": 11968266240, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:57.700805Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113262024192, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262024192, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262024192, + "device": 6, + "inactive_split_bytes_current": 11141630976, + "inactive_split_bytes_peak": 11968266240, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:01.589002Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9808153600, + "inactive_split_bytes_current_delta": -1333477376, + "inactive_split_bytes_peak": 11968266240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T141857701269Z.pickle", + "timestamp_utc": "2026-08-01T14:18:57.908439Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T141857701269Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T141857701269Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a51323ab1c436637012196b0868affad9f8b0504 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T141857701269Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1849c45436836d3cff3e61dae7437c9275e465a3905ad011d4e0659c97da9a42 +size 9212680 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T141751322372Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T141751322372Z.json new file mode 100644 index 0000000000000000000000000000000000000000..23d80027d11e98fa9740a640490ee263f8bba723 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T141751322372Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 31622094336, + "active_bytes_peak": 89629266944, + "active_bytes_peak_delta": 54162930688, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 89629266944, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 89629266944, + "device": 7, + "inactive_split_bytes_current": 8872512000, + "inactive_split_bytes_peak": 11609830400, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 94701092864, + "reserved_bytes_peak": 94701092864, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:17:51.321877Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 31622094336, + "allocated_bytes_peak": 89629266944, + "allocated_bytes_peak_delta": 54162930688, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:16:43.025526Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8872512000, + "inactive_split_bytes_current_delta": 8739693056, + "inactive_split_bytes_peak": 11609830400, + "inactive_split_bytes_peak_delta": 11449444352, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 94701092864, + "reserved_bytes_current_delta": 55457087488, + "reserved_bytes_peak": 94701092864, + "reserved_bytes_peak_delta": 55457087488, + "segment_count_current": 463, + "segment_count_current_delta": 392, + "segment_count_peak": 463, + "segment_count_peak_delta": 392, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T141751322372Z.pickle", + "timestamp_utc": "2026-08-01T14:17:51.434590Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T141751322372Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T141751322372Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f7fe6970a730fc9abe721867f6d548bb37a14d65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T141751322372Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85e0c7b3f0ff2c469d04929ac5f8ba579dc91678c523712f91a587d89e52236a +size 5446655 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T141752565757Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T141752565757Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76dff2095ce859ff1bb1a4180b9444d357a72ec7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T141752565757Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257347072, + "active_bytes_peak_delta": 23628080128, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113257347072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113257347072, + "device": 7, + "inactive_split_bytes_current": 11198253568, + "inactive_split_bytes_peak": 11972460032, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:52.565347Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257347072, + "allocated_bytes_peak_delta": 23628080128, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 89629266944, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 89629266944, + "device": 7, + "inactive_split_bytes_current": 8872512000, + "inactive_split_bytes_peak": 11609830400, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 94701092864, + "reserved_bytes_peak": 94701092864, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:17:51.455749Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11198253568, + "inactive_split_bytes_current_delta": 2325741568, + "inactive_split_bytes_peak": 11972460032, + "inactive_split_bytes_peak_delta": 362629632, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 21510488064, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 21510488064, + "segment_count_current": 538, + "segment_count_current_delta": 75, + "segment_count_peak": 538, + "segment_count_peak_delta": 75, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T141752565757Z.pickle", + "timestamp_utc": "2026-08-01T14:17:52.681963Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T141752565757Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T141752565757Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2b04bbed1c25b8ffb1bff944ed114c03faa9a802 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T141752565757Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c03021b651eda158136ecf4e21f93d919ed28ec9564636d7d3de62707950c55a +size 5589714 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T141753784121Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T141753784121Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ec314a6fcb868d23055f44864647d90765503ae7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T141753784121Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257347072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113257347072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113257347072, + "device": 7, + "inactive_split_bytes_current": 10837543424, + "inactive_split_bytes_peak": 11972460032, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:53.783722Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257347072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113257347072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113257347072, + "device": 7, + "inactive_split_bytes_current": 11198253568, + "inactive_split_bytes_peak": 11972460032, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:52.703530Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10837543424, + "inactive_split_bytes_current_delta": -360710144, + "inactive_split_bytes_peak": 11972460032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T141753784121Z.pickle", + "timestamp_utc": "2026-08-01T14:17:53.890596Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T141753784121Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T141753784121Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5c7228f1a7a837c42ce4d1b39f134bf1ebb71c63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T141753784121Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:388706c05e2eadd3a9e01eea3125c4f3c353e5ca80979c9d2a5942c95075a5eb +size 5606102 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T141754980495Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T141754980495Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b81f8f22cecd601e933c1ab9c96b4b91c96b138f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T141754980495Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257347072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113257347072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113257347072, + "device": 7, + "inactive_split_bytes_current": 10200009216, + "inactive_split_bytes_peak": 11972460032, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:54.980077Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257347072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113257347072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113257347072, + "device": 7, + "inactive_split_bytes_current": 10837543424, + "inactive_split_bytes_peak": 11972460032, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:53.905222Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10200009216, + "inactive_split_bytes_current_delta": -637534208, + "inactive_split_bytes_peak": 11972460032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T141754980495Z.pickle", + "timestamp_utc": "2026-08-01T14:17:55.082482Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T141754980495Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T141754980495Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bca1c8785618d76b1a95d5a74323d2932acc06a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T141754980495Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4d8ad285010107c317c4ac20e943e9516057a4384062156a3d3b8c7c1902b91 +size 5596157 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T141756762474Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T141756762474Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03d3d934771f63dd01aa3dd41b20c4d543f2d747 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T141756762474Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113257347072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113257347072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113257347072, + "device": 7, + "inactive_split_bytes_current": 9992391168, + "inactive_split_bytes_peak": 11972460032, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:56.762011Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113257347072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113257347072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113257347072, + "device": 7, + "inactive_split_bytes_current": 10200009216, + "inactive_split_bytes_peak": 11972460032, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:55.096754Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9992391168, + "inactive_split_bytes_current_delta": -207618048, + "inactive_split_bytes_peak": 11972460032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T141756762474Z.pickle", + "timestamp_utc": "2026-08-01T14:17:56.866611Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T141756762474Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T141756762474Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1f1aa736837097c32b45480d6e8fd065a082cca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T141756762474Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7aa80c51451352003672cab69020e47d0f0d8a3b8c0f65b56cbf8ff250a55ab2 +size 5595562 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T141757966147Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T141757966147Z.json new file mode 100644 index 0000000000000000000000000000000000000000..22834f579fb2ca1aadbd39ba3f4a5ae0ea8b1972 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T141757966147Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260258816, + "active_bytes_peak_delta": 2911744, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113260258816, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113260258816, + "device": 7, + "inactive_split_bytes_current": 10076277248, + "inactive_split_bytes_peak": 11972460032, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:57.965738Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260258816, + "allocated_bytes_peak_delta": 2911744, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113257347072, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113257347072, + "device": 7, + "inactive_split_bytes_current": 9992391168, + "inactive_split_bytes_peak": 11972460032, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:56.881602Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10076277248, + "inactive_split_bytes_current_delta": 83886080, + "inactive_split_bytes_peak": 11972460032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T141757966147Z.pickle", + "timestamp_utc": "2026-08-01T14:17:58.069266Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T141757966147Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T141757966147Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a843a400becf1d8a7ad660f581c26cd9074813f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T141757966147Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cba50bc1252dee4e5ecdcfa3f3d587f0764524272e90feaa54070a98bbd5ce53 +size 5595991 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T141759128774Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T141759128774Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6fa546d592460216556ec950b0cc14b22df94265 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T141759128774Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260258816, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113260258816, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113260258816, + "device": 7, + "inactive_split_bytes_current": 9908505088, + "inactive_split_bytes_peak": 11972460032, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:59.128361Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260258816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113260258816, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113260258816, + "device": 7, + "inactive_split_bytes_current": 10076277248, + "inactive_split_bytes_peak": 11972460032, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:58.084002Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9908505088, + "inactive_split_bytes_current_delta": -167772160, + "inactive_split_bytes_peak": 11972460032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T141759128774Z.pickle", + "timestamp_utc": "2026-08-01T14:17:59.230741Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T141759128774Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T141759128774Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c659af23ce3acc575170c42c82a214805a4d1b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T141759128774Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57e27c0d291ea0e1a11ab47740762292d6a6958764ddf13c9968b5bb6a19c843 +size 5595924 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T141800313895Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T141800313895Z.json new file mode 100644 index 0000000000000000000000000000000000000000..afb170d55a64d6fe5552ef83382f8ab06e004193 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T141800313895Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260258816, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113260258816, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113260258816, + "device": 7, + "inactive_split_bytes_current": 10231466496, + "inactive_split_bytes_peak": 11972460032, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:00.313490Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260258816, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113260258816, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113260258816, + "device": 7, + "inactive_split_bytes_current": 9908505088, + "inactive_split_bytes_peak": 11972460032, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:17:59.245555Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10231466496, + "inactive_split_bytes_current_delta": 322961408, + "inactive_split_bytes_peak": 11972460032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T141800313895Z.pickle", + "timestamp_utc": "2026-08-01T14:18:00.415379Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T141800313895Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T141800313895Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03bea9021fbda72507a410dd79e8e7d5ba97c723 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T141800313895Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f70cb052697d1dd782af39c50d447ed2edfa21889442f90fcd860c7d6e6b3e48 +size 5595646 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T141801477685Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T141801477685Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d3f643adc733629bf6a9110bc5a80768964e8a29 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T141801477685Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113262024704, + "active_bytes_peak_delta": 1765888, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113262024704, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113262024704, + "device": 7, + "inactive_split_bytes_current": 11147921920, + "inactive_split_bytes_peak": 11972460032, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:01.477286Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113262024704, + "allocated_bytes_peak_delta": 1765888, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113260258816, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113260258816, + "device": 7, + "inactive_split_bytes_current": 10231466496, + "inactive_split_bytes_peak": 11972460032, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:00.429976Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11147921920, + "inactive_split_bytes_current_delta": 916455424, + "inactive_split_bytes_peak": 11972460032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T141801477685Z.pickle", + "timestamp_utc": "2026-08-01T14:18:01.579508Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T141801477685Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T141801477685Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..619d2d100b5fc2276a6f4471393dfca5a19b9c49 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T141801477685Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f5bb055b9881ff2ed401b1d2601eaf698eab254e2fd67c15636c0d13180a3be +size 5595985 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T141857701252Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T141857701252Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d1285c33be9db1d6abb1ab776caac42a56069b06 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T141857701252Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906624, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113262024704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906624, + "active_bytes_peak": 113262024704, + "allocated_bytes_current": 66802906624, + "allocated_bytes_peak": 113262024704, + "device": 7, + "inactive_split_bytes_current": 9808153088, + "inactive_split_bytes_peak": 11972460032, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:57.700806Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906624, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113262024704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113262024704, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113262024704, + "device": 7, + "inactive_split_bytes_current": 11147921920, + "inactive_split_bytes_peak": 11972460032, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116211580928, + "reserved_bytes_peak": 116211580928, + "segment_count_current": 538, + "segment_count_peak": 538, + "timestamp_utc": "2026-08-01T14:18:01.593975Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9808153088, + "inactive_split_bytes_current_delta": -1339768832, + "inactive_split_bytes_peak": 11972460032, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116211580928, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116211580928, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 538, + "segment_count_current_delta": 0, + "segment_count_peak": 538, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T141857701252Z.pickle", + "timestamp_utc": "2026-08-01T14:18:57.895481Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T141857701252Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T141857701252Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e1d2f4d17503da2fb58df55c5d3aa668e714d05 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T141857701252Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3314ad1068b4d9eed936f9496c1d93428ec2f67c2d76f8e7202b336a7a645d98 +size 9214394 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..ffd3e4d3e2bd6a6372d0ea3eff8b701f30f48368 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 4096 +capacity: 1.25 +expert_dispatch_overlap: off +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6041 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 231 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T142142642624Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T142142642624Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a1adb255ebc74805fd7c99fede1c8acc433a79a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T142142642624Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441494528, + "active_bytes_current_delta": 37210039296, + "active_bytes_peak": 89523991552, + "active_bytes_peak_delta": 54057655296, + "after": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 89523991552, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 89523991552, + "device": 0, + "inactive_split_bytes_current": 7764083712, + "inactive_split_bytes_peak": 11713141248, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98656321536, + "reserved_bytes_peak": 98656321536, + "segment_count_current": 416, + "segment_count_peak": 416, + "timestamp_utc": "2026-08-01T14:21:42.642059Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441494528, + "allocated_bytes_current_delta": 37210039296, + "allocated_bytes_peak": 89523991552, + "allocated_bytes_peak_delta": 54057655296, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T14:20:34.485218Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7764083712, + "inactive_split_bytes_current_delta": 7629167616, + "inactive_split_bytes_peak": 11713141248, + "inactive_split_bytes_peak_delta": 11552509440, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98656321536, + "reserved_bytes_current_delta": 59410219008, + "reserved_bytes_peak": 98656321536, + "reserved_bytes_peak_delta": 59410219008, + "segment_count_current": 416, + "segment_count_current_delta": 344, + "segment_count_peak": 416, + "segment_count_peak_delta": 344, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T142142642624Z.pickle", + "timestamp_utc": "2026-08-01T14:21:42.771037Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T142142642624Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T142142642624Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d50b4a04273053aba63fbfc6c69ad65138491cb2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T142142642624Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e43dcace8dc17b7df0d6c17861756e30956ed5489569c27fe0b4beb2be816386 +size 7054993 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T142143977415Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T142143977415Z.json new file mode 100644 index 0000000000000000000000000000000000000000..46ea8ceecf19c79357bcc4830368f9e06f2e9683 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T142143977415Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441494528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113154749440, + "active_bytes_peak_delta": 23630757888, + "after": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113154749440, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113154749440, + "device": 0, + "inactive_split_bytes_current": 7409665024, + "inactive_split_bytes_peak": 14277013504, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:43.977008Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441494528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113154749440, + "allocated_bytes_peak_delta": 23630757888, + "before": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 89523991552, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 89523991552, + "device": 0, + "inactive_split_bytes_current": 7764083712, + "inactive_split_bytes_peak": 11713141248, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98656321536, + "reserved_bytes_peak": 98656321536, + "segment_count_current": 416, + "segment_count_peak": 416, + "timestamp_utc": "2026-08-01T14:21:42.790981Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7409665024, + "inactive_split_bytes_current_delta": -354418688, + "inactive_split_bytes_peak": 14277013504, + "inactive_split_bytes_peak_delta": 2563872256, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_current_delta": 18257805312, + "reserved_bytes_peak": 116914126848, + "reserved_bytes_peak_delta": 18257805312, + "segment_count_current": 476, + "segment_count_current_delta": 60, + "segment_count_peak": 476, + "segment_count_peak_delta": 60, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T142143977415Z.pickle", + "timestamp_utc": "2026-08-01T14:21:44.095392Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T142143977415Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T142143977415Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ab715260d84f02d339e3265088004daeb74f94c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T142143977415Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:682a8deb6d810bced59faff5150d8a72184c757daffdac055cd5f4deb3a5704e +size 5955046 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T142145308556Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T142145308556Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0bd4e748964a511d004eb19b0a3ab3574cc2fa7a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T142145308556Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441494528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113159705088, + "active_bytes_peak_delta": 4955648, + "after": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159705088, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159705088, + "device": 0, + "inactive_split_bytes_current": 8349189120, + "inactive_split_bytes_peak": 14930137600, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:45.308157Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441494528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113159705088, + "allocated_bytes_peak_delta": 4955648, + "before": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113154749440, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113154749440, + "device": 0, + "inactive_split_bytes_current": 7409665024, + "inactive_split_bytes_peak": 14277013504, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:44.111821Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8349189120, + "inactive_split_bytes_current_delta": 939524096, + "inactive_split_bytes_peak": 14930137600, + "inactive_split_bytes_peak_delta": 653124096, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116914126848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 476, + "segment_count_current_delta": 0, + "segment_count_peak": 476, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T142145308556Z.pickle", + "timestamp_utc": "2026-08-01T14:21:45.418567Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T142145308556Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T142145308556Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e7e1dca2d99b4398e2fdca17d8d946824ce7d046 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T142145308556Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4841da6f5311d56c9ec46bb6582480487e53db81690217199bb9406ecfe3e16d +size 5954260 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T142146628533Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T142146628533Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4e1b15ae927f17aae7dc7c354c5e33ed87982432 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T142146628533Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441494528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113159705088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159705088, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159705088, + "device": 0, + "inactive_split_bytes_current": 8999306240, + "inactive_split_bytes_peak": 14989780480, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:46.628118Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441494528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113159705088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159705088, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159705088, + "device": 0, + "inactive_split_bytes_current": 8349189120, + "inactive_split_bytes_peak": 14930137600, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:45.434503Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8999306240, + "inactive_split_bytes_current_delta": 650117120, + "inactive_split_bytes_peak": 14989780480, + "inactive_split_bytes_peak_delta": 59642880, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116914126848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 476, + "segment_count_current_delta": 0, + "segment_count_peak": 476, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T142146628533Z.pickle", + "timestamp_utc": "2026-08-01T14:21:46.739894Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T142146628533Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T142146628533Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..58791d25d3fb2750d3c0ec392bd9b3de968cb96c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T142146628533Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aefce29784e25184bea09aef3ae2de4cbb04b600092051b3ab9dafb6768a862 +size 5960026 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T142148212615Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T142148212615Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9754e3f897f5cd77b98685f2405d17aa4427ec45 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T142148212615Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441494528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113159705088, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159705088, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159705088, + "device": 0, + "inactive_split_bytes_current": 8175125504, + "inactive_split_bytes_peak": 14989780480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:48.212194Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441494528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113159705088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159705088, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159705088, + "device": 0, + "inactive_split_bytes_current": 8999306240, + "inactive_split_bytes_peak": 14989780480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:46.755840Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8175125504, + "inactive_split_bytes_current_delta": -824180736, + "inactive_split_bytes_peak": 14989780480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116914126848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 476, + "segment_count_current_delta": 0, + "segment_count_peak": 476, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T142148212615Z.pickle", + "timestamp_utc": "2026-08-01T14:21:48.323531Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T142148212615Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T142148212615Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e31e34d3b842f72a91efaac6b0fc919e75b90eb0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T142148212615Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:862d294d02ea413bdeb287536c09194b0248277cc103bb0dc6093f7aac94b575 +size 7649309 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T142149539453Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T142149539453Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2b9706ac79a36a450023244f908ec0fe9140c06 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T142149539453Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441494528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113159722496, + "active_bytes_peak_delta": 17408, + "after": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159722496, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159722496, + "device": 0, + "inactive_split_bytes_current": 8764425216, + "inactive_split_bytes_peak": 14989780480, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:49.539037Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441494528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113159722496, + "allocated_bytes_peak_delta": 17408, + "before": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159705088, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159705088, + "device": 0, + "inactive_split_bytes_current": 8175125504, + "inactive_split_bytes_peak": 14989780480, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:48.339265Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8764425216, + "inactive_split_bytes_current_delta": 589299712, + "inactive_split_bytes_peak": 14989780480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116914126848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 476, + "segment_count_current_delta": 0, + "segment_count_peak": 476, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T142149539453Z.pickle", + "timestamp_utc": "2026-08-01T14:21:49.650373Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T142149539453Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T142149539453Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5766d262b144e5d208122e3492254a28986d2aed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T142149539453Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb957781cc3e03e3c405e2a57bf4f7d1a104c227777b68c03c10bbe2caf30e8b +size 7649177 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T142150847754Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T142150847754Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a1814879573cd10282447dc36cf4a60ce14b977d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T142150847754Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441494528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113159722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159722496, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159722496, + "device": 0, + "inactive_split_bytes_current": 8588264448, + "inactive_split_bytes_peak": 14989780480, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:50.847355Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441494528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113159722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159722496, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159722496, + "device": 0, + "inactive_split_bytes_current": 8764425216, + "inactive_split_bytes_peak": 14989780480, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:49.666134Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8588264448, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 14989780480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116914126848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 476, + "segment_count_current_delta": 0, + "segment_count_peak": 476, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T142150847754Z.pickle", + "timestamp_utc": "2026-08-01T14:21:50.956724Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T142150847754Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T142150847754Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b4749bbd45da7d4240983f78e5ac70cfafff92a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T142150847754Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a3ba0369903934a691adcc7de3c78170640a6851c7f04e0eb3f7f99d615c20c +size 7649168 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T142152317707Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T142152317707Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ad14d7770ca80c826d24504a78a3ec49a45f744 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T142152317707Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441494528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113159722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159722496, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159722496, + "device": 0, + "inactive_split_bytes_current": 8995111936, + "inactive_split_bytes_peak": 14989780480, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:52.317298Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441494528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113159722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159722496, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159722496, + "device": 0, + "inactive_split_bytes_current": 8588264448, + "inactive_split_bytes_peak": 14989780480, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:50.972459Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8995111936, + "inactive_split_bytes_current_delta": 406847488, + "inactive_split_bytes_peak": 14989780480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116914126848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 476, + "segment_count_current_delta": 0, + "segment_count_peak": 476, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T142152317707Z.pickle", + "timestamp_utc": "2026-08-01T14:21:52.425473Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T142152317707Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T142152317707Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..46f65f0f98973547e07fa40f62c50dab41045ccf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T142152317707Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e9ed52922de427d6f3ff978dcb2c3f4e35d024a68300e42a56292aa0f765193 +size 7649229 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T142153627464Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T142153627464Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ac5a09f6b448f1ed9cc5044f5b0f56844d90e0b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T142153627464Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441494528, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113159722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159722496, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159722496, + "device": 0, + "inactive_split_bytes_current": 8470823936, + "inactive_split_bytes_peak": 14989780480, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:53.627040Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441494528, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113159722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159722496, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159722496, + "device": 0, + "inactive_split_bytes_current": 8995111936, + "inactive_split_bytes_peak": 14989780480, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:52.440818Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8470823936, + "inactive_split_bytes_current_delta": -524288000, + "inactive_split_bytes_peak": 14989780480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116914126848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 476, + "segment_count_current_delta": 0, + "segment_count_peak": 476, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T142153627464Z.pickle", + "timestamp_utc": "2026-08-01T14:21:53.734117Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T142153627464Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T142153627464Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..674d531e1dcb4d1a5f4e8e6424cf7734384eb250 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T142153627464Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9e1f9497fcd85f5e6a309e0ba774d516bbcf6aa89cb7a31868bfb3790f25297 +size 7649200 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T142250289377Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T142250289377Z.json new file mode 100644 index 0000000000000000000000000000000000000000..918914c6e3a11673c6f83aeabeb9c725433726c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T142250289377Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61167205376, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113159722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61167205376, + "active_bytes_peak": 113159722496, + "allocated_bytes_current": 61167205376, + "allocated_bytes_peak": 113159722496, + "device": 0, + "inactive_split_bytes_current": 10049979392, + "inactive_split_bytes_peak": 14989780480, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:22:50.288910Z", + "world_size": 8 + }, + "allocated_bytes_current": 61167205376, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113159722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441494528, + "active_bytes_peak": 113159722496, + "allocated_bytes_current": 72441494528, + "allocated_bytes_peak": 113159722496, + "device": 0, + "inactive_split_bytes_current": 8470823936, + "inactive_split_bytes_peak": 14989780480, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_peak": 116914126848, + "segment_count_current": 476, + "segment_count_peak": 476, + "timestamp_utc": "2026-08-01T14:21:53.749718Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10049979392, + "inactive_split_bytes_current_delta": 1579155456, + "inactive_split_bytes_peak": 14989780480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 116914126848, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116914126848, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 476, + "segment_count_current_delta": 0, + "segment_count_peak": 476, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T142250289377Z.pickle", + "timestamp_utc": "2026-08-01T14:22:50.504029Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T142250289377Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T142250289377Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..19540a99a8a6635bea8fe12bf798719d2358b208 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T142250289377Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70dc81584e4f8d91ed54da8190a6bb92bb9502e393b9a0373ccc7254d7a03c73 +size 12548544 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T142142642657Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T142142642657Z.json new file mode 100644 index 0000000000000000000000000000000000000000..efdac5ad5c461bbea9e0ad497b86601e62d76d52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T142142642657Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 37209644032, + "active_bytes_peak": 89522353152, + "active_bytes_peak_delta": 54056016896, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 89522353152, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 89522353152, + "device": 1, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_peak": 11747987968, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 98654224384, + "reserved_bytes_peak": 98654224384, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.642042Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 37209644032, + "allocated_bytes_peak": 89522353152, + "allocated_bytes_peak_delta": 54056016896, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:20:34.486990Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_current_delta": 8101422080, + "inactive_split_bytes_peak": 11747987968, + "inactive_split_bytes_peak_delta": 11587601920, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 98654224384, + "reserved_bytes_current_delta": 59410219008, + "reserved_bytes_peak": 98654224384, + "reserved_bytes_peak_delta": 59410219008, + "segment_count_current": 415, + "segment_count_current_delta": 344, + "segment_count_peak": 415, + "segment_count_peak_delta": 344, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T142142642657Z.pickle", + "timestamp_utc": "2026-08-01T14:21:42.761628Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T142142642657Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T142142642657Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..612f3185e7bcd114997b2af98cfdd0ca5f64b930 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T142142642657Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c84dff7ccdcb82e31aa30966ad19ce8039d003b03f9d5f7c68fec894f51e9730 +size 7045933 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T142143977414Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T142143977414Z.json new file mode 100644 index 0000000000000000000000000000000000000000..429da7454ca566dc1e3aa04c1b6a85a80eb0baa5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T142143977414Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157124096, + "active_bytes_peak_delta": 23634770944, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 1, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_peak": 13927183872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:43.976997Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157124096, + "allocated_bytes_peak_delta": 23634770944, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 89522353152, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 89522353152, + "device": 1, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_peak": 11747987968, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 98654224384, + "reserved_bytes_peak": 98654224384, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.780106Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_current_delta": 1293942784, + "inactive_split_bytes_peak": 13927183872, + "inactive_split_bytes_peak_delta": 2179195904, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 18253611008, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 18253611008, + "segment_count_current": 474, + "segment_count_current_delta": 59, + "segment_count_peak": 474, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T142143977414Z.pickle", + "timestamp_utc": "2026-08-01T14:21:44.095323Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T142143977414Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T142143977414Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..405f796aa99f927908150eeef68e5015167f35d9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T142143977414Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47863a2ad20638c890b57b018d02aad2b88c41200bd56add46f54db52ad04e9c +size 5953876 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T142145308556Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T142145308556Z.json new file mode 100644 index 0000000000000000000000000000000000000000..16c843234d8089eceb69cda77d0d9cbe7b948ae2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T142145308556Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 1, + "inactive_split_bytes_current": 8706100224, + "inactive_split_bytes_peak": 14421322752, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:45.308153Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 1, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_peak": 13927183872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:44.110993Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8706100224, + "inactive_split_bytes_current_delta": -822083584, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 494138880, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T142145308556Z.pickle", + "timestamp_utc": "2026-08-01T14:21:45.419430Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T142145308556Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T142145308556Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5bb9c9d6942fee0673f9e77d31fb13ab8090ed7a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T142145308556Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:789a9b7dd15ab42796c833ae76f499bf6acc2c3df3dc4b227104f201401cdd71 +size 5952694 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T142146628505Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T142146628505Z.json new file mode 100644 index 0000000000000000000000000000000000000000..32e2372f48bebbad63ad54c5186e4472faa434ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T142146628505Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157931008, + "active_bytes_peak_delta": 806912, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 1, + "inactive_split_bytes_current": 8643185664, + "inactive_split_bytes_peak": 14421322752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:46.628086Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157931008, + "allocated_bytes_peak_delta": 806912, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 1, + "inactive_split_bytes_current": 8706100224, + "inactive_split_bytes_peak": 14421322752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:45.434605Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8643185664, + "inactive_split_bytes_current_delta": -62914560, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T142146628505Z.pickle", + "timestamp_utc": "2026-08-01T14:21:46.741701Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T142146628505Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T142146628505Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6d13b2d5974113ba1a0267ca736f404c32c566f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T142146628505Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcb0ce25c9bc9fd7ece152187d2f1b66881765547caf16ae642059424d22ba81 +size 6382462 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T142148212458Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T142148212458Z.json new file mode 100644 index 0000000000000000000000000000000000000000..705882acc865ddb01b2b49f041ed8916ccf53fc9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T142148212458Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157931008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 1, + "inactive_split_bytes_current": 10289449984, + "inactive_split_bytes_peak": 14421322752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:48.212040Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157931008, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 1, + "inactive_split_bytes_current": 8643185664, + "inactive_split_bytes_peak": 14421322752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:46.756971Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10289449984, + "inactive_split_bytes_current_delta": 1646264320, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T142148212458Z.pickle", + "timestamp_utc": "2026-08-01T14:21:48.324650Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T142148212458Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T142148212458Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..06c3dd5ea3042ae9e267a2ed5a8c9cd2d0847032 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T142148212458Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:740a080dea5e3690e526b6161dca264bda9d13413a0164410412b1a9d93d6602 +size 6527752 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T142149539440Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T142149539440Z.json new file mode 100644 index 0000000000000000000000000000000000000000..716e9549bc2b0e5f98959671f77519d623458a92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T142149539440Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113160049664, + "active_bytes_peak_delta": 2118656, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113160049664, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113160049664, + "device": 1, + "inactive_split_bytes_current": 8699808768, + "inactive_split_bytes_peak": 14421322752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:49.539027Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113160049664, + "allocated_bytes_peak_delta": 2118656, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 1, + "inactive_split_bytes_current": 10289449984, + "inactive_split_bytes_peak": 14421322752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:48.339641Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8699808768, + "inactive_split_bytes_current_delta": -1589641216, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T142149539440Z.pickle", + "timestamp_utc": "2026-08-01T14:21:49.650636Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T142149539440Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T142149539440Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4446551045b05aad3af5bc83b08a047872e3d210 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T142149539440Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:627b3c57447f08b5485c4990c661ef3667e482da7714b747029dcb4ce5ddc8f4 +size 6382398 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T142150847761Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T142150847761Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c7a317e7b898c6193e046c69f22a6752a9756f00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T142150847761Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 5463040, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 1, + "inactive_split_bytes_current": 8936786944, + "inactive_split_bytes_peak": 14421322752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:50.847351Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 5463040, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113160049664, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113160049664, + "device": 1, + "inactive_split_bytes_current": 8699808768, + "inactive_split_bytes_peak": 14421322752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:49.665740Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8936786944, + "inactive_split_bytes_current_delta": 236978176, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T142150847761Z.pickle", + "timestamp_utc": "2026-08-01T14:21:50.957033Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T142150847761Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T142150847761Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a31bf28c3339dec0dda598f5cb9d75b65c00d6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T142150847761Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6af568cc080146202bdcf13737239c61a53ca0f349d45868cecc6cd5774fd1e5 +size 6479124 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T142152317672Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T142152317672Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f183da364d44de2ee85f03fd039ae5cb7c3e67b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T142152317672Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 1, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_peak": 14421322752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:52.317273Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 1, + "inactive_split_bytes_current": 8936786944, + "inactive_split_bytes_peak": 14421322752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:50.971999Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_current_delta": 589299712, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T142152317672Z.pickle", + "timestamp_utc": "2026-08-01T14:21:52.426093Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T142152317672Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T142152317672Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b061dd6a7854b54cf71363302e7df7dadaa12dcc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T142152317672Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c39548b155dc8f94aa697eb1fd80f92668be192ec17f1b93fc92702d408aaa04 +size 6618579 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T142153627411Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T142153627411Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d564651a62f4b3e51189588fadc5e755ae2126cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T142153627411Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 1, + "inactive_split_bytes_current": 8588659712, + "inactive_split_bytes_peak": 14421322752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:53.627006Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 1, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_peak": 14421322752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:52.440770Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8588659712, + "inactive_split_bytes_current_delta": -937426944, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T142153627411Z.pickle", + "timestamp_utc": "2026-08-01T14:21:53.735361Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T142153627411Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T142153627411Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..86098b3790ab78748bb74492f3d859f99b5f483c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T142153627411Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f64c0b668b6ebb2fd6122b0868c5aeea8674abcfe563a39e144d22d44d00471b +size 6661029 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T142250289487Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T142250289487Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef887e7da4ef2c7f4e859e0e1d777a5ab05323aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T142250289487Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166810112, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166810112, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 61166810112, + "allocated_bytes_peak": 113165512704, + "device": 1, + "inactive_split_bytes_current": 9930836992, + "inactive_split_bytes_peak": 14439408640, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:22:50.288911Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166810112, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 1, + "inactive_split_bytes_current": 8588659712, + "inactive_split_bytes_peak": 14421322752, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:53.750178Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9930836992, + "inactive_split_bytes_current_delta": 1342177280, + "inactive_split_bytes_peak": 14439408640, + "inactive_split_bytes_peak_delta": 18085888, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T142250289487Z.pickle", + "timestamp_utc": "2026-08-01T14:22:50.503309Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T142250289487Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T142250289487Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4736189b2ac09236b9e16b07fdd1f10228ab3246 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T142250289487Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ab0d081c51835672d22754ca9d3d44f147246c30d6396f48acd1cfc0f385ec1 +size 9833261 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T142142642691Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T142142642691Z.json new file mode 100644 index 0000000000000000000000000000000000000000..66073ff74d6bd6bd70355e1e919d14743874778b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T142142642691Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 37209644032, + "active_bytes_peak": 89522353152, + "active_bytes_peak_delta": 54056016896, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 89522353152, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 89522353152, + "device": 2, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_peak": 11747987968, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98654224384, + "reserved_bytes_peak": 98654224384, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.642069Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 37209644032, + "allocated_bytes_peak": 89522353152, + "allocated_bytes_peak_delta": 54056016896, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:20:34.485989Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_current_delta": 8101422080, + "inactive_split_bytes_peak": 11747987968, + "inactive_split_bytes_peak_delta": 11587601920, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98654224384, + "reserved_bytes_current_delta": 59410219008, + "reserved_bytes_peak": 98654224384, + "reserved_bytes_peak_delta": 59410219008, + "segment_count_current": 415, + "segment_count_current_delta": 344, + "segment_count_peak": 415, + "segment_count_peak_delta": 344, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T142142642691Z.pickle", + "timestamp_utc": "2026-08-01T14:21:42.758969Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T142142642691Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T142142642691Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..28ff15a7b5e0ad8af5146632834b3240fe1f2a43 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T142142642691Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9419b4652b67db44286d0f2f473467c062e21c2dffe8d74cc18f4ce2c372a1a6 +size 7045938 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T142143977496Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T142143977496Z.json new file mode 100644 index 0000000000000000000000000000000000000000..750f88b80c5a16e095f91298bdc35e40d950b97a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T142143977496Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157124096, + "active_bytes_peak_delta": 23634770944, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 2, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_peak": 13927183872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:43.977064Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157124096, + "allocated_bytes_peak_delta": 23634770944, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 89522353152, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 89522353152, + "device": 2, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_peak": 11747987968, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98654224384, + "reserved_bytes_peak": 98654224384, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.775978Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_current_delta": 1293942784, + "inactive_split_bytes_peak": 13927183872, + "inactive_split_bytes_peak_delta": 2179195904, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 18253611008, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 18253611008, + "segment_count_current": 474, + "segment_count_current_delta": 59, + "segment_count_peak": 474, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T142143977496Z.pickle", + "timestamp_utc": "2026-08-01T14:21:44.092715Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T142143977496Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T142143977496Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e5c20ca9c577909e0d1d3e044d469e673c4ed977 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T142143977496Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dab061a0959dc4b34ff2056341a40cdb54e4d9f2edf534758e09841aecc82559 +size 5953879 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T142145308592Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T142145308592Z.json new file mode 100644 index 0000000000000000000000000000000000000000..126504dc4fbbf0d572cb2efa3e79a3cdecdde1bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T142145308592Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 2, + "inactive_split_bytes_current": 8706100224, + "inactive_split_bytes_peak": 14421322752, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:45.308195Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 2, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_peak": 13927183872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:44.106789Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8706100224, + "inactive_split_bytes_current_delta": -822083584, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 494138880, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T142145308592Z.pickle", + "timestamp_utc": "2026-08-01T14:21:45.424477Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T142145308592Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T142145308592Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9bf78af13f933cd04c4ca401a88a3b82c3634173 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T142145308592Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a7847a478c47198da04e5b1534d9afb8487381e31189a90edbab9c2736c8c49 +size 5952705 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T142146628533Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T142146628533Z.json new file mode 100644 index 0000000000000000000000000000000000000000..88f8fb889a6fa7923c92efacd70791f36526dc21 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T142146628533Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157931008, + "active_bytes_peak_delta": 806912, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 2, + "inactive_split_bytes_current": 8643185664, + "inactive_split_bytes_peak": 14421322752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:46.628135Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157931008, + "allocated_bytes_peak_delta": 806912, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 2, + "inactive_split_bytes_current": 8706100224, + "inactive_split_bytes_peak": 14421322752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:45.440523Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8643185664, + "inactive_split_bytes_current_delta": -62914560, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T142146628533Z.pickle", + "timestamp_utc": "2026-08-01T14:21:46.734902Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T142146628533Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T142146628533Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a0442a986de141408603bd5067062257f5816aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T142146628533Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7ad173bdb53f8cc3999aa70a06056c148a59f6008168c6bd7d6dfc0eb6dffab +size 6382467 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T142148212500Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T142148212500Z.json new file mode 100644 index 0000000000000000000000000000000000000000..299625c4388ecceb13adfab4e638ac3fe1fab78e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T142148212500Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157931008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 2, + "inactive_split_bytes_current": 10289449984, + "inactive_split_bytes_peak": 14421322752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:48.212093Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157931008, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 2, + "inactive_split_bytes_current": 8643185664, + "inactive_split_bytes_peak": 14421322752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:46.748539Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10289449984, + "inactive_split_bytes_current_delta": 1646264320, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T142148212500Z.pickle", + "timestamp_utc": "2026-08-01T14:21:48.321856Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T142148212500Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T142148212500Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4bfc2eeb6da138434005df46683f336a5f961618 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T142148212500Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c1ef6cb124b784981e5168771d3c4943600295f6850d251d8ba73049caec2ae +size 6527755 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T142149539453Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T142149539453Z.json new file mode 100644 index 0000000000000000000000000000000000000000..50aaf75d608b12e51ce8e72310e0f0685e72055f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T142149539453Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113160049664, + "active_bytes_peak_delta": 2118656, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113160049664, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113160049664, + "device": 2, + "inactive_split_bytes_current": 8699808768, + "inactive_split_bytes_peak": 14421322752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:49.539041Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113160049664, + "allocated_bytes_peak_delta": 2118656, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 2, + "inactive_split_bytes_current": 10289449984, + "inactive_split_bytes_peak": 14421322752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:48.335711Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8699808768, + "inactive_split_bytes_current_delta": -1589641216, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T142149539453Z.pickle", + "timestamp_utc": "2026-08-01T14:21:49.645858Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T142149539453Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T142149539453Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c1aef377b815054771427090809fe7e590dbe065 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T142149539453Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d91e7e4a82c9f3b9de192f8bbd320a452da6bf0843f1d96867ae28d6d568bcd9 +size 6382398 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T142150847781Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T142150847781Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8d6eeee37cb79b652a555a248f397cc0846cd0ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T142150847781Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 5463040, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 2, + "inactive_split_bytes_current": 8936786944, + "inactive_split_bytes_peak": 14421322752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:50.847369Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 5463040, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113160049664, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113160049664, + "device": 2, + "inactive_split_bytes_current": 8699808768, + "inactive_split_bytes_peak": 14421322752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:49.659531Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8936786944, + "inactive_split_bytes_current_delta": 236978176, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T142150847781Z.pickle", + "timestamp_utc": "2026-08-01T14:21:50.953269Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T142150847781Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T142150847781Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e4980e399a6341a97e9d8e7a31d9d2820b29bb2c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T142150847781Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b03405d17675ae40dd745b0a5a8cda5f94231b16c833c56784f5ae32a28cfda3 +size 6479124 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T142152317702Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T142152317702Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d142b32ed90f5869900bede2d1e50c5b7beca256 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T142152317702Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 2, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_peak": 14421322752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:52.317300Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 2, + "inactive_split_bytes_current": 8936786944, + "inactive_split_bytes_peak": 14421322752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:50.966695Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_current_delta": 589299712, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T142152317702Z.pickle", + "timestamp_utc": "2026-08-01T14:21:52.421520Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T142152317702Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T142152317702Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..48131784d3a00aa4852c6dce9956d4f96865a8a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T142152317702Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35c983653fef456bdd87624acb079ddcb88cdb2d4619f25e11e4c491704184d9 +size 6618582 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T142153627461Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T142153627461Z.json new file mode 100644 index 0000000000000000000000000000000000000000..233f244031a6c9fa32ab1b8d30a560444f18dffc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T142153627461Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 2, + "inactive_split_bytes_current": 8588659712, + "inactive_split_bytes_peak": 14421322752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:53.627048Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 2, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_peak": 14421322752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:52.434888Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8588659712, + "inactive_split_bytes_current_delta": -937426944, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T142153627461Z.pickle", + "timestamp_utc": "2026-08-01T14:21:53.730810Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T142153627461Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T142153627461Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9e8c5382391bcd12973b5fecc2071e829e0e6325 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T142153627461Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:963b973eb19511c14b9e4c5900ba78fba6d79d53bdbafadc9092ffe717fd4a86 +size 6661030 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T142250289391Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T142250289391Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1502aff12c5610ca0e4600f2067b4c278ff83d94 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T142250289391Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166810112, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166810112, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 61166810112, + "allocated_bytes_peak": 113165512704, + "device": 2, + "inactive_split_bytes_current": 9930836992, + "inactive_split_bytes_peak": 14441505792, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:22:50.288907Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166810112, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 2, + "inactive_split_bytes_current": 8588659712, + "inactive_split_bytes_peak": 14421322752, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:53.744069Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9930836992, + "inactive_split_bytes_current_delta": 1342177280, + "inactive_split_bytes_peak": 14441505792, + "inactive_split_bytes_peak_delta": 20183040, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T142250289391Z.pickle", + "timestamp_utc": "2026-08-01T14:22:50.488238Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T142250289391Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T142250289391Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2f4d9e63d94742c99f8d8833d2a717ec2c91862 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T142250289391Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71eb83cf5edbcdb734c731a6498b85ffcc993bc43b8530b22bdb0d91d2d46c84 +size 9833264 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T142142642735Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T142142642735Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d6a250a27c900b04d917a656d4953c290437cb00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T142142642735Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 37209644032, + "active_bytes_peak": 89522353152, + "active_bytes_peak_delta": 54056016896, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 89522353152, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 89522353152, + "device": 3, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_peak": 11747987968, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 98654224384, + "reserved_bytes_peak": 98654224384, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.642136Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 37209644032, + "allocated_bytes_peak": 89522353152, + "allocated_bytes_peak_delta": 54056016896, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:20:34.485214Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_current_delta": 8101422080, + "inactive_split_bytes_peak": 11747987968, + "inactive_split_bytes_peak_delta": 11587601920, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 98654224384, + "reserved_bytes_current_delta": 59410219008, + "reserved_bytes_peak": 98654224384, + "reserved_bytes_peak_delta": 59410219008, + "segment_count_current": 415, + "segment_count_current_delta": 344, + "segment_count_peak": 415, + "segment_count_peak_delta": 344, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T142142642735Z.pickle", + "timestamp_utc": "2026-08-01T14:21:42.764000Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T142142642735Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T142142642735Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..afd1c8a9f0e6558c46c38b82a3aa8f39aa14332a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T142142642735Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b77e50696654a3cefdd63f7696f99425a93b3cbfc04bc9822142626b34be0c1 +size 7045929 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T142143977440Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T142143977440Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ef06d82a39afee783d3cfd745814efcd6abb6b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T142143977440Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157124096, + "active_bytes_peak_delta": 23634770944, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 3, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_peak": 13927183872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:43.977020Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157124096, + "allocated_bytes_peak_delta": 23634770944, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 89522353152, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 89522353152, + "device": 3, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_peak": 11747987968, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 98654224384, + "reserved_bytes_peak": 98654224384, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.782388Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_current_delta": 1293942784, + "inactive_split_bytes_peak": 13927183872, + "inactive_split_bytes_peak_delta": 2179195904, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 18253611008, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 18253611008, + "segment_count_current": 474, + "segment_count_current_delta": 59, + "segment_count_peak": 474, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T142143977440Z.pickle", + "timestamp_utc": "2026-08-01T14:21:44.093239Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T142143977440Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T142143977440Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..25377827d7b5630aacfabe43a5e68ff53d0d6ff1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T142143977440Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:729f3b2a42348314750e0d815593e100a0f6f9fa99098f06eee1cb276abcc8d0 +size 5953869 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T142145308592Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T142145308592Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f209a3c2d3011363fe7c6245e7d58263ac97aa57 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T142145308592Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 3, + "inactive_split_bytes_current": 8706100224, + "inactive_split_bytes_peak": 14421322752, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:45.308170Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 3, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_peak": 13927183872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:44.109335Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8706100224, + "inactive_split_bytes_current_delta": -822083584, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 494138880, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T142145308592Z.pickle", + "timestamp_utc": "2026-08-01T14:21:45.418080Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T142145308592Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T142145308592Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ce8510b3ad8592f1490c315ab0f62d5485bfb85 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T142145308592Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f847744e682e82d8c8e3e0c9b709a7bb21a2fd6b2412a4690df2c637262f00f0 +size 5952663 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T142146628529Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T142146628529Z.json new file mode 100644 index 0000000000000000000000000000000000000000..015ff87f224648f76e10c8292b57bb68dc4fdfc7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T142146628529Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157931008, + "active_bytes_peak_delta": 806912, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 3, + "inactive_split_bytes_current": 8643185664, + "inactive_split_bytes_peak": 14421322752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:46.628094Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157931008, + "allocated_bytes_peak_delta": 806912, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 3, + "inactive_split_bytes_current": 8706100224, + "inactive_split_bytes_peak": 14421322752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:45.433662Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8643185664, + "inactive_split_bytes_current_delta": -62914560, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T142146628529Z.pickle", + "timestamp_utc": "2026-08-01T14:21:46.737561Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T142146628529Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T142146628529Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f7c28afc87eaa58337de533870ed305e2180b3a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T142146628529Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93b37f9b62be0efb8c1161f3f67cc6b1311097a5cecd3e5274fffafe654b8802 +size 6382472 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T142148212487Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T142148212487Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d00335e0a0104441544e8e839eb86f5dfa2280bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T142148212487Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157931008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 3, + "inactive_split_bytes_current": 10289449984, + "inactive_split_bytes_peak": 14421322752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:48.212067Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157931008, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 3, + "inactive_split_bytes_current": 8643185664, + "inactive_split_bytes_peak": 14421322752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:46.753209Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10289449984, + "inactive_split_bytes_current_delta": 1646264320, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T142148212487Z.pickle", + "timestamp_utc": "2026-08-01T14:21:48.321609Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T142148212487Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T142148212487Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f62636b0dead20e4d39648dea3bdd4509728fc19 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T142148212487Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e33d3945bb78709040b8cf19c4ee900a1610123c0b77ad4a4a61233ab49f6eed +size 6527729 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T142149539450Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T142149539450Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b2a72dd669756a2e42d52cf99fd7a61033842770 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T142149539450Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113160049664, + "active_bytes_peak_delta": 2118656, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113160049664, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113160049664, + "device": 3, + "inactive_split_bytes_current": 8699808768, + "inactive_split_bytes_peak": 14421322752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:49.539032Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113160049664, + "allocated_bytes_peak_delta": 2118656, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 3, + "inactive_split_bytes_current": 10289449984, + "inactive_split_bytes_peak": 14421322752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:48.337658Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8699808768, + "inactive_split_bytes_current_delta": -1589641216, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T142149539450Z.pickle", + "timestamp_utc": "2026-08-01T14:21:49.646950Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T142149539450Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T142149539450Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..88840f70d6b1541d90d4893ac5d621c8171beaa0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T142149539450Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99c7752e8dcf6dca3a5ad26e778c2303e5294da186292131338bf065372f72e3 +size 6382428 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T142150847857Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T142150847857Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a80bfbd565b86f25c11c141261b99360628374b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T142150847857Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 5463040, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 3, + "inactive_split_bytes_current": 8938884096, + "inactive_split_bytes_peak": 14421322752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:50.847453Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 5463040, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113160049664, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113160049664, + "device": 3, + "inactive_split_bytes_current": 8699808768, + "inactive_split_bytes_peak": 14421322752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:49.662249Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8938884096, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T142150847857Z.pickle", + "timestamp_utc": "2026-08-01T14:21:50.954541Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T142150847857Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T142150847857Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..410fe325f533c41338bd915d794346fcf1ad50d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T142150847857Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da32d50fddeb4835e5c13c2631b520c2bb1d8a0bc230b0f03f53f807364d96d7 +size 6479150 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T142152317701Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T142152317701Z.json new file mode 100644 index 0000000000000000000000000000000000000000..29cfce9d35235ad04cc07a416b629c64ff709e97 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T142152317701Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 3, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_peak": 14421322752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:52.317290Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 3, + "inactive_split_bytes_current": 8938884096, + "inactive_split_bytes_peak": 14421322752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:50.969753Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_current_delta": 587202560, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T142152317701Z.pickle", + "timestamp_utc": "2026-08-01T14:21:52.429554Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T142152317701Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T142152317701Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ed4e7dbb8a282abdebca6d27047d0fa7420926c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T142152317701Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f1ae9803d6056fcd562091a248d322ee8ea6ae5330ea691a9507333e66d7b75 +size 6618579 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T142153627449Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T142153627449Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f7ea8ba084d463406338807cca70a9530aa7553a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T142153627449Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 3, + "inactive_split_bytes_current": 8588659712, + "inactive_split_bytes_peak": 14421322752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:53.627040Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 3, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_peak": 14421322752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:52.449190Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8588659712, + "inactive_split_bytes_current_delta": -937426944, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T142153627449Z.pickle", + "timestamp_utc": "2026-08-01T14:21:53.735097Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T142153627449Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T142153627449Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ce6a796baa43480111a492b69813dcfe175a001 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T142153627449Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0dbf1ac1329f0d06e3135074c3bdd3b097097deaf0e4c26c1d9a09ccab1be3a +size 6661031 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T142250289404Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T142250289404Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9cd15d887bdb2b33e897208aeda3259359aa7d48 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T142250289404Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166810112, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166810112, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 61166810112, + "allocated_bytes_peak": 113165512704, + "device": 3, + "inactive_split_bytes_current": 9930836992, + "inactive_split_bytes_peak": 14441505792, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:22:50.288920Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166810112, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 3, + "inactive_split_bytes_current": 8588659712, + "inactive_split_bytes_peak": 14421322752, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:53.750175Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9930836992, + "inactive_split_bytes_current_delta": 1342177280, + "inactive_split_bytes_peak": 14441505792, + "inactive_split_bytes_peak_delta": 20183040, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T142250289404Z.pickle", + "timestamp_utc": "2026-08-01T14:22:50.489463Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T142250289404Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T142250289404Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..faff8a80078288dbe27fc89f3e06ac427feb8d94 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T142250289404Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bc3937aac9e84fef7ebdb7b12d44da2c13f7b3ea87613e49226e86dc15744f5 +size 9833270 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T142142642656Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T142142642656Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3aed43e424a3820839fa91357fee5adcd9fac8c4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T142142642656Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442698752, + "active_bytes_current_delta": 37210801152, + "active_bytes_peak": 89526022656, + "active_bytes_peak_delta": 54059244032, + "after": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 89526022656, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 89526022656, + "device": 4, + "inactive_split_bytes_current": 8226350080, + "inactive_split_bytes_peak": 10596736512, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 99035906048, + "reserved_bytes_peak": 99035906048, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.642084Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442698752, + "allocated_bytes_current_delta": 37210801152, + "allocated_bytes_peak": 89526022656, + "allocated_bytes_peak_delta": 54059244032, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:20:34.485715Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8226350080, + "inactive_split_bytes_current_delta": 8093973504, + "inactive_split_bytes_peak": 10596736512, + "inactive_split_bytes_peak_delta": 10443326976, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 99035906048, + "reserved_bytes_current_delta": 59791900672, + "reserved_bytes_peak": 99035906048, + "reserved_bytes_peak_delta": 59791900672, + "segment_count_current": 415, + "segment_count_current_delta": 344, + "segment_count_peak": 415, + "segment_count_peak_delta": 344, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T142142642656Z.pickle", + "timestamp_utc": "2026-08-01T14:21:42.760843Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T142142642656Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T142142642656Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db929a7e6b12cffcb6b8a9e9b125997d790dad2e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T142142642656Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa1972dbaf7f223576d7e11855e8b500c0c7477ecbd15a3f12bc43607f3c27a2 +size 7072292 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T142143977419Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T142143977419Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2cfc2d7d54e6109167a0ad98dbba505487b4b03 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T142143977419Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442698752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113148867072, + "active_bytes_peak_delta": 23622844416, + "after": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113148867072, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113148867072, + "device": 4, + "inactive_split_bytes_current": 8721277952, + "inactive_split_bytes_peak": 14311974912, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:43.977012Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442698752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113148867072, + "allocated_bytes_peak_delta": 23622844416, + "before": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 89526022656, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 89526022656, + "device": 4, + "inactive_split_bytes_current": 8226350080, + "inactive_split_bytes_peak": 10596736512, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 99035906048, + "reserved_bytes_peak": 99035906048, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.777764Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8721277952, + "inactive_split_bytes_current_delta": 494927872, + "inactive_split_bytes_peak": 14311974912, + "inactive_split_bytes_peak_delta": 3715238400, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116878475264, + "reserved_bytes_current_delta": 17842569216, + "reserved_bytes_peak": 116878475264, + "reserved_bytes_peak_delta": 17842569216, + "segment_count_current": 475, + "segment_count_current_delta": 60, + "segment_count_peak": 475, + "segment_count_peak_delta": 60, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T142143977419Z.pickle", + "timestamp_utc": "2026-08-01T14:21:44.090934Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T142143977419Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T142143977419Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de5f5c1e69a0b940410bb66502e1aea983c174f3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T142143977419Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5697036986c9208787ccb4412fa07de18879f8f5c21930a1aebf239751273ca +size 5955665 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T142145308533Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T142145308533Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ec68d2bc3521859c5fb77852b2ad6309317b305 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T142145308533Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442698752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113153782272, + "active_bytes_peak_delta": 4915200, + "after": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113153782272, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113153782272, + "device": 4, + "inactive_split_bytes_current": 8872272896, + "inactive_split_bytes_peak": 14311974912, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:45.308145Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442698752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113153782272, + "allocated_bytes_peak_delta": 4915200, + "before": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113148867072, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113148867072, + "device": 4, + "inactive_split_bytes_current": 8721277952, + "inactive_split_bytes_peak": 14311974912, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:44.104862Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8872272896, + "inactive_split_bytes_current_delta": 150994944, + "inactive_split_bytes_peak": 14311974912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116878475264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116878475264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 475, + "segment_count_current_delta": 0, + "segment_count_peak": 475, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T142145308533Z.pickle", + "timestamp_utc": "2026-08-01T14:21:45.417330Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T142145308533Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T142145308533Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..41f4edf437fcd20bd8b56c0c046922a17283c9a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T142145308533Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae4edcdf7a48689d61c6815c0cfd3ba8518e498672a6203fa4e3a120fcbb72eb +size 5961633 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T142146628501Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T142146628501Z.json new file mode 100644 index 0000000000000000000000000000000000000000..92550907deef2047c934b8c7b763034e47220b0f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T142146628501Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442698752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113153782272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113153782272, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113153782272, + "device": 4, + "inactive_split_bytes_current": 9167971328, + "inactive_split_bytes_peak": 14434226688, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:46.628101Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442698752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113153782272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113153782272, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113153782272, + "device": 4, + "inactive_split_bytes_current": 8872272896, + "inactive_split_bytes_peak": 14311974912, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:45.433225Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9167971328, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 14434226688, + "inactive_split_bytes_peak_delta": 122251776, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116878475264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116878475264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 475, + "segment_count_current_delta": 0, + "segment_count_peak": 475, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T142146628501Z.pickle", + "timestamp_utc": "2026-08-01T14:21:46.732859Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T142146628501Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T142146628501Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..63453dc9751254c2518873ab5bebdd1776081fd0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T142146628501Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5041c649795232858e2e69c826b7759f0fc7ce15f20f4523fad2c5a41b2c1a0f +size 5960704 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T142148212490Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T142148212490Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ff7c68a3af3f66f5af450fd31e54453fa17816e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T142148212490Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442698752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113153782272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113153782272, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113153782272, + "device": 4, + "inactive_split_bytes_current": 9167971328, + "inactive_split_bytes_peak": 15336177664, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:48.212079Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442698752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113153782272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113153782272, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113153782272, + "device": 4, + "inactive_split_bytes_current": 9167971328, + "inactive_split_bytes_peak": 14434226688, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:46.746388Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9167971328, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15336177664, + "inactive_split_bytes_peak_delta": 901950976, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116878475264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116878475264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 475, + "segment_count_current_delta": 0, + "segment_count_peak": 475, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T142148212490Z.pickle", + "timestamp_utc": "2026-08-01T14:21:48.319343Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T142148212490Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T142148212490Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1f8a0757efbeaf6addbf380c9173b48ebcfd9d37 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T142148212490Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82efb4d1d3e53990090462533f723b89fcaafa34c7869ffb4f17f83b5811101d +size 5978277 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T142149539490Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T142149539490Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee8006cfeeb58402ba1975832e1f32bb8a472452 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T142149539490Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442698752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113153782272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113153782272, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113153782272, + "device": 4, + "inactive_split_bytes_current": 9872614400, + "inactive_split_bytes_peak": 15336177664, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:49.539084Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442698752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113153782272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113153782272, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113153782272, + "device": 4, + "inactive_split_bytes_current": 9167971328, + "inactive_split_bytes_peak": 15336177664, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:48.333049Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9872614400, + "inactive_split_bytes_current_delta": 704643072, + "inactive_split_bytes_peak": 15336177664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116878475264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116878475264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 475, + "segment_count_current_delta": 0, + "segment_count_peak": 475, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T142149539490Z.pickle", + "timestamp_utc": "2026-08-01T14:21:49.645868Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T142149539490Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T142149539490Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fe68a276d5868eacc388d9d8feb66656ae104baa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T142149539490Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c41a064bce42ff1d8c20eeb14671120b80b77eb923601c3873e68825e6ba0fdc +size 5960012 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T142150847758Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T142150847758Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bd8620d92b64c4f84aa711a4038e87c73f56e2fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T142150847758Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442698752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113153782272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113153782272, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113153782272, + "device": 4, + "inactive_split_bytes_current": 9226691584, + "inactive_split_bytes_peak": 15336177664, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:50.847356Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442698752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113153782272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113153782272, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113153782272, + "device": 4, + "inactive_split_bytes_current": 9872614400, + "inactive_split_bytes_peak": 15336177664, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:49.659558Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9226691584, + "inactive_split_bytes_current_delta": -645922816, + "inactive_split_bytes_peak": 15336177664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116878475264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116878475264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 475, + "segment_count_current_delta": 0, + "segment_count_peak": 475, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T142150847758Z.pickle", + "timestamp_utc": "2026-08-01T14:21:50.950893Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T142150847758Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T142150847758Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ccee83c64852e837a56dd14253cc5b4844a6768 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T142150847758Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f451952a4649889c805db07354c6a5d5450ad942f3b20c9547dde2993928a334 +size 5959960 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T142152317663Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T142152317663Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e136613daf90fe9bd2160b5685146e3547ee604a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T142152317663Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442698752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113154044416, + "active_bytes_peak_delta": 262144, + "after": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113154044416, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113154044416, + "device": 4, + "inactive_split_bytes_current": 9579013120, + "inactive_split_bytes_peak": 15336177664, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:52.317267Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442698752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113154044416, + "allocated_bytes_peak_delta": 262144, + "before": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113153782272, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113153782272, + "device": 4, + "inactive_split_bytes_current": 9226691584, + "inactive_split_bytes_peak": 15336177664, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:50.964299Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9579013120, + "inactive_split_bytes_current_delta": 352321536, + "inactive_split_bytes_peak": 15336177664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116878475264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116878475264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 475, + "segment_count_current_delta": 0, + "segment_count_peak": 475, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T142152317663Z.pickle", + "timestamp_utc": "2026-08-01T14:21:52.419504Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T142152317663Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T142152317663Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f5bf2cb72d15e3426873ba1e9c9d806e83eb97a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T142152317663Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2aa035f749a2dec13ab9d8a55c830d95056cc4d8b806d018783d79b103dcee57 +size 5959889 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T142153627448Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T142153627448Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef0b8075cf106c3774d4fc152e80e562cb5aa917 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T142153627448Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442698752, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113154839040, + "active_bytes_peak_delta": 794624, + "after": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113154839040, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113154839040, + "device": 4, + "inactive_split_bytes_current": 8928896000, + "inactive_split_bytes_peak": 15336177664, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:53.627036Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442698752, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113154839040, + "allocated_bytes_peak_delta": 794624, + "before": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113154044416, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113154044416, + "device": 4, + "inactive_split_bytes_current": 9579013120, + "inactive_split_bytes_peak": 15336177664, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:52.432738Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8928896000, + "inactive_split_bytes_current_delta": -650117120, + "inactive_split_bytes_peak": 15336177664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116878475264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116878475264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 475, + "segment_count_current_delta": 0, + "segment_count_peak": 475, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T142153627448Z.pickle", + "timestamp_utc": "2026-08-01T14:21:53.729812Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T142153627448Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T142153627448Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9d847d3254b33cb56213aacdfe6ec3a6ac84ef6f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T142153627448Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03f448af46ad486c1dd94a80edfaf8abde33ab4a29e982474d23289961e0f366 +size 5977995 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T142250289392Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T142250289392Z.json new file mode 100644 index 0000000000000000000000000000000000000000..74cbd20da1df87ad91d6e05cdf1c942fbb7d6fb8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T142250289392Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61168409600, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113156862976, + "active_bytes_peak_delta": 2023936, + "after": { + "active_bytes_current": 61168409600, + "active_bytes_peak": 113156862976, + "allocated_bytes_current": 61168409600, + "allocated_bytes_peak": 113156862976, + "device": 4, + "inactive_split_bytes_current": 9690162176, + "inactive_split_bytes_peak": 15336177664, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:22:50.288903Z", + "world_size": 8 + }, + "allocated_bytes_current": 61168409600, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113156862976, + "allocated_bytes_peak_delta": 2023936, + "before": { + "active_bytes_current": 72442698752, + "active_bytes_peak": 113154839040, + "allocated_bytes_current": 72442698752, + "allocated_bytes_peak": 113154839040, + "device": 4, + "inactive_split_bytes_current": 8928896000, + "inactive_split_bytes_peak": 15336177664, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 116878475264, + "reserved_bytes_peak": 116878475264, + "segment_count_current": 475, + "segment_count_peak": 475, + "timestamp_utc": "2026-08-01T14:21:53.743045Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9690162176, + "inactive_split_bytes_current_delta": 761266176, + "inactive_split_bytes_peak": 15336177664, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 116878475264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116878475264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 475, + "segment_count_current_delta": 0, + "segment_count_peak": 475, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T142250289392Z.pickle", + "timestamp_utc": "2026-08-01T14:22:50.486822Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T142250289392Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T142250289392Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14087de47d54ea7d61dbddf19910dce12d01b04f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T142250289392Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38cd9056be5836ed2d7bf3021f72158ffd8445d14eeaeaf6f4eeac10e0b0c683 +size 9750792 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T142142642773Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T142142642773Z.json new file mode 100644 index 0000000000000000000000000000000000000000..09665439692411993df1faa26ee1b636cfae0ed1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T142142642773Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 37209644032, + "active_bytes_peak": 89522353152, + "active_bytes_peak_delta": 54056016896, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 89522353152, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 89522353152, + "device": 5, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_peak": 11747987968, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 98654224384, + "reserved_bytes_peak": 98654224384, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.642158Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 37209644032, + "allocated_bytes_peak": 89522353152, + "allocated_bytes_peak_delta": 54056016896, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:20:34.485045Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_current_delta": 8101422080, + "inactive_split_bytes_peak": 11747987968, + "inactive_split_bytes_peak_delta": 11587601920, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 98654224384, + "reserved_bytes_current_delta": 59410219008, + "reserved_bytes_peak": 98654224384, + "reserved_bytes_peak_delta": 59410219008, + "segment_count_current": 415, + "segment_count_current_delta": 344, + "segment_count_peak": 415, + "segment_count_peak_delta": 344, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T142142642773Z.pickle", + "timestamp_utc": "2026-08-01T14:21:42.763236Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T142142642773Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T142142642773Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..13010fbd863dd6209a054ac342d9789a1c1334af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T142142642773Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:011e43f7de1e5cc97fc37b73e99f65e679dc6a1d8c915507b78f12366130fd8a +size 7045935 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T142143977435Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T142143977435Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9b3d124d16841eb72c1f029327f8e98888ea34e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T142143977435Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157124096, + "active_bytes_peak_delta": 23634770944, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 5, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_peak": 13927183872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:43.977027Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157124096, + "allocated_bytes_peak_delta": 23634770944, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 89522353152, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 89522353152, + "device": 5, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_peak": 11747987968, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 98654224384, + "reserved_bytes_peak": 98654224384, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.780147Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_current_delta": 1293942784, + "inactive_split_bytes_peak": 13927183872, + "inactive_split_bytes_peak_delta": 2179195904, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 18253611008, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 18253611008, + "segment_count_current": 474, + "segment_count_current_delta": 59, + "segment_count_peak": 474, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T142143977435Z.pickle", + "timestamp_utc": "2026-08-01T14:21:44.091725Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T142143977435Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T142143977435Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b7280c6e325d25fd3b9fe4c911f1aec35a24ca08 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T142143977435Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1b061412980fb0b1329df00041801e0338773daece9f4737024ebb884239d25 +size 5953869 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T142145311059Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T142145311059Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4828c76bf00123699d9e0ae2bd607dba95e54522 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T142145311059Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 5, + "inactive_split_bytes_current": 8708197376, + "inactive_split_bytes_peak": 14421322752, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:45.308158Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 5, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_peak": 13927183872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:44.105898Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8708197376, + "inactive_split_bytes_current_delta": -819986432, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 494138880, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T142145311059Z.pickle", + "timestamp_utc": "2026-08-01T14:21:45.418013Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T142145311059Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T142145311059Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..367801b45034421dde35becd646aa76cde9e3c5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T142145311059Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeba4b0b3fee81b09721e6e6c232cf4ea7105021368db06cb5a9c7adac168b68 +size 5952695 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T142146628527Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T142146628527Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c790500bc0d162b276f67fc452d6a2e430f669e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T142146628527Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157931008, + "active_bytes_peak_delta": 806912, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 5, + "inactive_split_bytes_current": 8645282816, + "inactive_split_bytes_peak": 14421322752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:46.628109Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157931008, + "allocated_bytes_peak_delta": 806912, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 5, + "inactive_split_bytes_current": 8708197376, + "inactive_split_bytes_peak": 14421322752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:45.431461Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8645282816, + "inactive_split_bytes_current_delta": -62914560, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T142146628527Z.pickle", + "timestamp_utc": "2026-08-01T14:21:46.734769Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T142146628527Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T142146628527Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..69070fb58779e11f227add0d7cfb95d98e3f2cb2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T142146628527Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb90170e7a711410cc39f2f7dd472b28296043523fda16e28c0f389fbe395be3 +size 6382464 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T142148212489Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T142148212489Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a8cdc38753a5acd27a7bf06bba255b0276f8c0ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T142148212489Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157931008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 5, + "inactive_split_bytes_current": 10291547136, + "inactive_split_bytes_peak": 14421322752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:48.212064Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157931008, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 5, + "inactive_split_bytes_current": 8645282816, + "inactive_split_bytes_peak": 14421322752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:46.748482Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10291547136, + "inactive_split_bytes_current_delta": 1646264320, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T142148212489Z.pickle", + "timestamp_utc": "2026-08-01T14:21:48.320172Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T142148212489Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T142148212489Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f302bf9fafe21872d5cb5c360bd390b71a950ac7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T142148212489Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fda1911ffe8441177e9ecb2498450fd6dd30e2dc8c5e1726aef3fe8ad707c89a +size 6527726 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T142149539427Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T142149539427Z.json new file mode 100644 index 0000000000000000000000000000000000000000..737e232dde5dff2dd1b894a80c73837ad2d410f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T142149539427Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113160049664, + "active_bytes_peak_delta": 2118656, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113160049664, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113160049664, + "device": 5, + "inactive_split_bytes_current": 8701905920, + "inactive_split_bytes_peak": 14421322752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:49.539026Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113160049664, + "allocated_bytes_peak_delta": 2118656, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 5, + "inactive_split_bytes_current": 10291547136, + "inactive_split_bytes_peak": 14421322752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:48.333735Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8701905920, + "inactive_split_bytes_current_delta": -1589641216, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T142149539427Z.pickle", + "timestamp_utc": "2026-08-01T14:21:49.644966Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T142149539427Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T142149539427Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11fb54eb58822827315a81ae6dbf7eb789a693bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T142149539427Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b670af1605a5e5d7022d35265e6f2106f5efc66398512e4a3da387ea20b0365 +size 6382395 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T142150847769Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T142150847769Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4468efdeb97a89c139c69fd7981567abc5965e1b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T142150847769Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 5463040, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 5, + "inactive_split_bytes_current": 8938884096, + "inactive_split_bytes_peak": 14421322752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:50.847369Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 5463040, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113160049664, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113160049664, + "device": 5, + "inactive_split_bytes_current": 8701905920, + "inactive_split_bytes_peak": 14421322752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:49.658416Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8938884096, + "inactive_split_bytes_current_delta": 236978176, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T142150847769Z.pickle", + "timestamp_utc": "2026-08-01T14:21:50.952449Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T142150847769Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T142150847769Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4918c58b3fdb679d892e65694d6e1a3d4872ccda --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T142150847769Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebb3da221d63fd17b349c5ba60ab350160dd8bac13dd46b63f1faa970a2e5e4c +size 6479098 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T142152317700Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T142152317700Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d0ef1f27072c7841223507ac7f005f57ffaf38d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T142152317700Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 5, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_peak": 14421322752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:52.317288Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 5, + "inactive_split_bytes_current": 8938884096, + "inactive_split_bytes_peak": 14421322752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:50.965828Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_current_delta": 587202560, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T142152317700Z.pickle", + "timestamp_utc": "2026-08-01T14:21:52.423601Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T142152317700Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T142152317700Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e944ececa7e22d5bf7db4c0778d021c71d1c3fa1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T142152317700Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d2616cfcf8387b4a78acd4bd3e5651ff69449c4090eb54830a7258a35e61b35 +size 6618553 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T142153627444Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T142153627444Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5b4c430e5b0b9dd67e6f0f921e8e94c867693eb0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T142153627444Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 5, + "inactive_split_bytes_current": 8590756864, + "inactive_split_bytes_peak": 14421322752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:53.627040Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 5, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_peak": 14421322752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:52.436975Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8590756864, + "inactive_split_bytes_current_delta": -935329792, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T142153627444Z.pickle", + "timestamp_utc": "2026-08-01T14:21:53.729556Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T142153627444Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T142153627444Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1a492363ddf8da43a12e0052866dc287b1fe0015 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T142153627444Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af5e0f21495944e85d2dcc3e84b391446e5b9383d7c95b0c679238f75d29d0dc +size 6661066 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T142250289496Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T142250289496Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5d37cdfdb1ee2356a96db78b763e5e9c4d0d8d23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T142250289496Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166810112, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166810112, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 61166810112, + "allocated_bytes_peak": 113165512704, + "device": 5, + "inactive_split_bytes_current": 9932934144, + "inactive_split_bytes_peak": 14441505792, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:22:50.288910Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166810112, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 5, + "inactive_split_bytes_current": 8590756864, + "inactive_split_bytes_peak": 14421322752, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:53.742597Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9932934144, + "inactive_split_bytes_current_delta": 1342177280, + "inactive_split_bytes_peak": 14441505792, + "inactive_split_bytes_peak_delta": 20183040, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T142250289496Z.pickle", + "timestamp_utc": "2026-08-01T14:22:50.496567Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T142250289496Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T142250289496Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..980589602d479844da3ffeedc7b47fc2845acfcd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T142250289496Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1a10dc5bf00d37b6aa261e0ccdc4d6cb9c784398a86df4b6f28c12c340d5e8e +size 9833255 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T142142643309Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T142142643309Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd052f82bcbd4aa6dcdb8a08e4e7007abf278936 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T142142643309Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 37209644032, + "active_bytes_peak": 89522353152, + "active_bytes_peak_delta": 54056016896, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 89522353152, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 89522353152, + "device": 6, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_peak": 11747987968, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98654224384, + "reserved_bytes_peak": 98654224384, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.642476Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 37209644032, + "allocated_bytes_peak": 89522353152, + "allocated_bytes_peak_delta": 54056016896, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:20:34.485083Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_current_delta": 8101422080, + "inactive_split_bytes_peak": 11747987968, + "inactive_split_bytes_peak_delta": 11587601920, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98654224384, + "reserved_bytes_current_delta": 59410219008, + "reserved_bytes_peak": 98654224384, + "reserved_bytes_peak_delta": 59410219008, + "segment_count_current": 415, + "segment_count_current_delta": 344, + "segment_count_peak": 415, + "segment_count_peak_delta": 344, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T142142643309Z.pickle", + "timestamp_utc": "2026-08-01T14:21:42.767058Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T142142643309Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T142142643309Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f29276ac2f70d78e1ba77062bdb34176e079a174 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T142142643309Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9216f67faf0e66869ff4e8e03d513b7098c5e15b2279b3116f8b02a7bff71f33 +size 7045933 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T142143977818Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T142143977818Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0318a46ea95b71546b84b6ac60199082579b4664 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T142143977818Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157124096, + "active_bytes_peak_delta": 23634770944, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 6, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_peak": 13927183872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:43.977258Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157124096, + "allocated_bytes_peak_delta": 23634770944, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 89522353152, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 89522353152, + "device": 6, + "inactive_split_bytes_current": 8234241024, + "inactive_split_bytes_peak": 11747987968, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98654224384, + "reserved_bytes_peak": 98654224384, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.785881Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_current_delta": 1293942784, + "inactive_split_bytes_peak": 13927183872, + "inactive_split_bytes_peak_delta": 2179195904, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 18253611008, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 18253611008, + "segment_count_current": 474, + "segment_count_current_delta": 59, + "segment_count_peak": 474, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T142143977818Z.pickle", + "timestamp_utc": "2026-08-01T14:21:44.098357Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T142143977818Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T142143977818Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2db0257d7f8d522876bc3266d0e86a1eed23fd71 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T142143977818Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7215bebd8a4cfea8fe0bfb7949cbd42dcdff77bf2edc76dbbb98693a3bb1095e +size 5953872 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T142145308896Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T142145308896Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f8e061689f2be2f774f2e96c5168da90e3e3a294 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T142145308896Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157124096, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 6, + "inactive_split_bytes_current": 8706100224, + "inactive_split_bytes_peak": 14421322752, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:45.308391Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157124096, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 6, + "inactive_split_bytes_current": 9528183808, + "inactive_split_bytes_peak": 13927183872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:44.114107Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8706100224, + "inactive_split_bytes_current_delta": -822083584, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 494138880, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T142145308896Z.pickle", + "timestamp_utc": "2026-08-01T14:21:45.419923Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T142145308896Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T142145308896Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cff0de17799bfbc1bb7cc3c5ff9a9bb1f03f4a23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T142145308896Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56c4bac5b10245980ae5688e46d23acfc476d574d6b5ed4d2e9aaa9330b54137 +size 5952704 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T142146628834Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T142146628834Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eee4f9875fc3cfbfda95e4da2f5de2308137f4f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T142146628834Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157931008, + "active_bytes_peak_delta": 806912, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 6, + "inactive_split_bytes_current": 8643185664, + "inactive_split_bytes_peak": 14421322752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:46.628295Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157931008, + "allocated_bytes_peak_delta": 806912, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157124096, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157124096, + "device": 6, + "inactive_split_bytes_current": 8706100224, + "inactive_split_bytes_peak": 14421322752, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:45.434928Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8643185664, + "inactive_split_bytes_current_delta": -62914560, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T142146628834Z.pickle", + "timestamp_utc": "2026-08-01T14:21:46.743497Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T142146628834Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T142146628834Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..627e351aa24729348eea9d27a604143597754fe7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T142146628834Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cfb9c0e647e8666ae761c0de53775b4abcdd8d148e8f5da139e8281308a8cf4 +size 6382459 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T142148212957Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T142148212957Z.json new file mode 100644 index 0000000000000000000000000000000000000000..523a9082d1e96d760a73fb7c4fce0d8156f0f8b4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T142148212957Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157931008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 6, + "inactive_split_bytes_current": 10289449984, + "inactive_split_bytes_peak": 14421322752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:48.212449Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157931008, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 6, + "inactive_split_bytes_current": 8643185664, + "inactive_split_bytes_peak": 14421322752, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:46.759111Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10289449984, + "inactive_split_bytes_current_delta": 1646264320, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T142148212957Z.pickle", + "timestamp_utc": "2026-08-01T14:21:48.332500Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T142148212957Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T142148212957Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..248e97e0ecd938d7fed51a8ca12d9f26abb34415 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T142148212957Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f83c68bf9002dc32b0c5cbde1dd41ca8e34c98c98049760221f3872e942e2d2 +size 6527757 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T142149539706Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T142149539706Z.json new file mode 100644 index 0000000000000000000000000000000000000000..472c32d29fb2725f4ffede4a640cdc39f406cb37 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T142149539706Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113160049664, + "active_bytes_peak_delta": 2118656, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113160049664, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113160049664, + "device": 6, + "inactive_split_bytes_current": 8699808768, + "inactive_split_bytes_peak": 14421322752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:49.539200Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113160049664, + "allocated_bytes_peak_delta": 2118656, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113157931008, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113157931008, + "device": 6, + "inactive_split_bytes_current": 10289449984, + "inactive_split_bytes_peak": 14421322752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:48.350336Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8699808768, + "inactive_split_bytes_current_delta": -1589641216, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T142149539706Z.pickle", + "timestamp_utc": "2026-08-01T14:21:49.646055Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T142149539706Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T142149539706Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf29d7c2aa3687c5caa6c355473d405eb188a229 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T142149539706Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:662ed6894cb2001a19aa4ddc39a7f3988e831e583f3c3881c924f172f82c3564 +size 6382395 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T142150848074Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T142150848074Z.json new file mode 100644 index 0000000000000000000000000000000000000000..861b202234bae03543452814bc70b184d6e89d22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T142150848074Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 5463040, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 6, + "inactive_split_bytes_current": 8936786944, + "inactive_split_bytes_peak": 14421322752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:50.847558Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 5463040, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113160049664, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113160049664, + "device": 6, + "inactive_split_bytes_current": 8699808768, + "inactive_split_bytes_peak": 14421322752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:49.660779Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8936786944, + "inactive_split_bytes_current_delta": 236978176, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T142150848074Z.pickle", + "timestamp_utc": "2026-08-01T14:21:50.953361Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T142150848074Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T142150848074Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..52e4c794e871018e3fb81e2d341890cd54ba377e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T142150848074Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70186ded57044766d1b3da30f95bfee986e1c657c8a40f4f101d3d8a55998812 +size 6479120 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T142152317892Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T142152317892Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cb68048a276d81d81dc0cc06674c45103ed4d6fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T142152317892Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 6, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_peak": 14421322752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:52.317402Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 6, + "inactive_split_bytes_current": 8936786944, + "inactive_split_bytes_peak": 14421322752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:50.967923Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_current_delta": 589299712, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T142152317892Z.pickle", + "timestamp_utc": "2026-08-01T14:21:52.422078Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T142152317892Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T142152317892Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df0b42afc0baff3bc67237e014ad5b58b37875ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T142152317892Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7dc28da5b2bb10cf80a6ce924bf6b4210797ce2c1beff7eb64ea3e4f98f5233 +size 6618577 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T142153627715Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T142153627715Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5281bad5daed0ab27ad2b632eefbbca056fd7d6d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T142153627715Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 6, + "inactive_split_bytes_current": 8588659712, + "inactive_split_bytes_peak": 14421322752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:53.627210Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 6, + "inactive_split_bytes_current": 9526086656, + "inactive_split_bytes_peak": 14421322752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:52.436342Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8588659712, + "inactive_split_bytes_current_delta": -937426944, + "inactive_split_bytes_peak": 14421322752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T142153627715Z.pickle", + "timestamp_utc": "2026-08-01T14:21:53.730012Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T142153627715Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T142153627715Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ab77e4ff4e27126ebd833597196c30bcd3cead0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T142153627715Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2794a34d4090204de778e4b44a8d45edb7ef33315a7dac33a880e6c286662fd9 +size 6661022 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T142250289460Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T142250289460Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6263b4d3aa00da1e0fbb383420a23727ea3a1d71 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T142250289460Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166810112, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113165512704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166810112, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 61166810112, + "allocated_bytes_peak": 113165512704, + "device": 6, + "inactive_split_bytes_current": 9930836992, + "inactive_split_bytes_peak": 14441505792, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:22:50.288907Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166810112, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113165512704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 113165512704, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 113165512704, + "device": 6, + "inactive_split_bytes_current": 8588659712, + "inactive_split_bytes_peak": 14421322752, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:53.748855Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9930836992, + "inactive_split_bytes_current_delta": 1342177280, + "inactive_split_bytes_peak": 14441505792, + "inactive_split_bytes_peak_delta": 20183040, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T142250289460Z.pickle", + "timestamp_utc": "2026-08-01T14:22:50.506389Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T142250289460Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T142250289460Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db02df722baa604bd4e43a2a7049fc435ff42348 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T142250289460Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dffd3313a463f8d9491c81245966bb57b8eb8389fc81dcdda25c247b5120f281 +size 9833252 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T142142642695Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T142142642695Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f4ddcdcb071a323ce4f278aada85f1cebdbc8bb5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T142142642695Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099776, + "active_bytes_current_delta": 37209644544, + "active_bytes_peak": 89522353152, + "active_bytes_peak_delta": 54056016896, + "after": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 89522353152, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 89522353152, + "device": 7, + "inactive_split_bytes_current": 8234240512, + "inactive_split_bytes_peak": 11747987968, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 98654224384, + "reserved_bytes_peak": 98654224384, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.642096Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099776, + "allocated_bytes_current_delta": 37209644544, + "allocated_bytes_peak": 89522353152, + "allocated_bytes_peak_delta": 54056016896, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:20:34.488069Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8234240512, + "inactive_split_bytes_current_delta": 8101421568, + "inactive_split_bytes_peak": 11747987968, + "inactive_split_bytes_peak_delta": 11587601920, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 98654224384, + "reserved_bytes_current_delta": 59410219008, + "reserved_bytes_peak": 98654224384, + "reserved_bytes_peak_delta": 59410219008, + "segment_count_current": 415, + "segment_count_current_delta": 344, + "segment_count_peak": 415, + "segment_count_peak_delta": 344, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T142142642695Z.pickle", + "timestamp_utc": "2026-08-01T14:21:42.761017Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T142142642695Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T142142642695Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..65f9152b664f473442735406ba43ad05960112af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T142142642695Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58fb554d7068263658a1683ea8cda17d1a84d66a72de6e177b9da3f82c5f781f +size 7047526 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T142143977435Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T142143977435Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1bd776f93b48f4afbb15e95559cf84d434277adb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T142143977435Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157124608, + "active_bytes_peak_delta": 23634771456, + "after": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113157124608, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113157124608, + "device": 7, + "inactive_split_bytes_current": 9528183296, + "inactive_split_bytes_peak": 13927183360, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:43.977012Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157124608, + "allocated_bytes_peak_delta": 23634771456, + "before": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 89522353152, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 89522353152, + "device": 7, + "inactive_split_bytes_current": 8234240512, + "inactive_split_bytes_peak": 11747987968, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 98654224384, + "reserved_bytes_peak": 98654224384, + "segment_count_current": 415, + "segment_count_peak": 415, + "timestamp_utc": "2026-08-01T14:21:42.778027Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9528183296, + "inactive_split_bytes_current_delta": 1293942784, + "inactive_split_bytes_peak": 13927183360, + "inactive_split_bytes_peak_delta": 2179195392, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 18253611008, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 18253611008, + "segment_count_current": 474, + "segment_count_current_delta": 59, + "segment_count_peak": 474, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T142143977435Z.pickle", + "timestamp_utc": "2026-08-01T14:21:44.092646Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T142143977435Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T142143977435Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..32f80f04b05002ab7cf0a6decd01f3d2d179bb61 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T142143977435Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88963d165a0da7b7a4886de8f650071b8027cb7598afefc7f0021630e0cc43a0 +size 5955619 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T142145308588Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T142145308588Z.json new file mode 100644 index 0000000000000000000000000000000000000000..325839028a4f6ea1b280233a76e72d7f0ee3df0c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T142145308588Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157124608, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113157124608, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113157124608, + "device": 7, + "inactive_split_bytes_current": 8706099712, + "inactive_split_bytes_peak": 14421322240, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:45.308177Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157124608, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113157124608, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113157124608, + "device": 7, + "inactive_split_bytes_current": 9528183296, + "inactive_split_bytes_peak": 13927183360, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:44.106708Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8706099712, + "inactive_split_bytes_current_delta": -822083584, + "inactive_split_bytes_peak": 14421322240, + "inactive_split_bytes_peak_delta": 494138880, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T142145308588Z.pickle", + "timestamp_utc": "2026-08-01T14:21:45.415297Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T142145308588Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T142145308588Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6dc2505dc8ad8da74034125da82bdcbc4b7ac835 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T142145308588Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bb4e242aa5d5f07a76fcdfe38352052d318775254d8af50fb2ba240092f7f49 +size 5954490 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T142146628576Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T142146628576Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15b35d0756cc828e70d862f188693edf8461d082 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T142146628576Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157931520, + "active_bytes_peak_delta": 806912, + "after": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113157931520, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113157931520, + "device": 7, + "inactive_split_bytes_current": 8643185152, + "inactive_split_bytes_peak": 14421322240, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:46.628157Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157931520, + "allocated_bytes_peak_delta": 806912, + "before": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113157124608, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113157124608, + "device": 7, + "inactive_split_bytes_current": 8706099712, + "inactive_split_bytes_peak": 14421322240, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:45.428847Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8643185152, + "inactive_split_bytes_current_delta": -62914560, + "inactive_split_bytes_peak": 14421322240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T142146628576Z.pickle", + "timestamp_utc": "2026-08-01T14:21:46.734697Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T142146628576Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T142146628576Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f844f0ff5c01a2f0fff975fdb45f86fbd3f720c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T142146628576Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:effc92d41d5b973352f4e516ee8bdff49496b6ba40cfe3fda658c0b1c171bdbf +size 6384227 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T142148212522Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T142148212522Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5536ec484ed3f1191236d9cb18ab33e2a801e70f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T142148212522Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113157931520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113157931520, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113157931520, + "device": 7, + "inactive_split_bytes_current": 10289449472, + "inactive_split_bytes_peak": 14421322240, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:48.212098Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157931520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113157931520, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113157931520, + "device": 7, + "inactive_split_bytes_current": 8643185152, + "inactive_split_bytes_peak": 14421322240, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:46.748451Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10289449472, + "inactive_split_bytes_current_delta": 1646264320, + "inactive_split_bytes_peak": 14421322240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T142148212522Z.pickle", + "timestamp_utc": "2026-08-01T14:21:48.319344Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T142148212522Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T142148212522Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72d1ed769e8b56e32b058e51be7a45b7e2c852df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T142148212522Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cd8099937a16909d3846f4ec7b016532eb09b6aae84294021b94b1277fd8a66 +size 6529584 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T142149539440Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T142149539440Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3557e46108f0e86c668a9b17c8a7c2bde8754ece --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T142149539440Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113160050176, + "active_bytes_peak_delta": 2118656, + "after": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113160050176, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113160050176, + "device": 7, + "inactive_split_bytes_current": 8699808256, + "inactive_split_bytes_peak": 14421322240, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:49.539028Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113160050176, + "allocated_bytes_peak_delta": 2118656, + "before": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113157931520, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113157931520, + "device": 7, + "inactive_split_bytes_current": 10289449472, + "inactive_split_bytes_peak": 14421322240, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:48.333079Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8699808256, + "inactive_split_bytes_current_delta": -1589641216, + "inactive_split_bytes_peak": 14421322240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T142149539440Z.pickle", + "timestamp_utc": "2026-08-01T14:21:49.650477Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T142149539440Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T142149539440Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ea09885677f29ad8be3620275992f4d08967dd9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T142149539440Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1a4a5e534caf718e14ac38078253caeeba0a49312506dc4b8ea7f40198b5a8b +size 6384154 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T142150847795Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T142150847795Z.json new file mode 100644 index 0000000000000000000000000000000000000000..410a0d16ca34792ceb051405415bdc15b08511ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T142150847795Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165513216, + "active_bytes_peak_delta": 5463040, + "after": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113165513216, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113165513216, + "device": 7, + "inactive_split_bytes_current": 8936786432, + "inactive_split_bytes_peak": 14421322240, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:50.847396Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165513216, + "allocated_bytes_peak_delta": 5463040, + "before": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113160050176, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113160050176, + "device": 7, + "inactive_split_bytes_current": 8699808256, + "inactive_split_bytes_peak": 14421322240, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:49.666502Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8936786432, + "inactive_split_bytes_current_delta": 236978176, + "inactive_split_bytes_peak": 14421322240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T142150847795Z.pickle", + "timestamp_utc": "2026-08-01T14:21:50.952453Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T142150847795Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T142150847795Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..47305ee3fbe5f3050a942a40787570bd892f50aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T142150847795Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f6f39e2e37f47c8f62a9cc3db337a54e6e4a515e5a82e056a98006b7ba19048 +size 6480950 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T142152317747Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T142152317747Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4591b22b5b06d476a3b48e0502b63c2dfb2fde33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T142152317747Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165513216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113165513216, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113165513216, + "device": 7, + "inactive_split_bytes_current": 9526086144, + "inactive_split_bytes_peak": 14421322240, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:52.317347Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165513216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113165513216, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113165513216, + "device": 7, + "inactive_split_bytes_current": 8936786432, + "inactive_split_bytes_peak": 14421322240, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:50.965610Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9526086144, + "inactive_split_bytes_current_delta": 589299712, + "inactive_split_bytes_peak": 14421322240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T142152317747Z.pickle", + "timestamp_utc": "2026-08-01T14:21:52.421650Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T142152317747Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T142152317747Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..95506e29fb96688b82c9061e9b750bfb4714d8cf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T142152317747Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69e015de3e855d4eb605013febe4f4153428601a1bef1ac474a9a30340fa3569 +size 6620380 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T142153627484Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T142153627484Z.json new file mode 100644 index 0000000000000000000000000000000000000000..460219ecd7d0f380242c261172fbf32a0d106bfe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T142153627484Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099776, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113165513216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113165513216, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113165513216, + "device": 7, + "inactive_split_bytes_current": 8588659200, + "inactive_split_bytes_peak": 14421322240, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:53.627089Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099776, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165513216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113165513216, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113165513216, + "device": 7, + "inactive_split_bytes_current": 9526086144, + "inactive_split_bytes_peak": 14421322240, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:52.434840Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8588659200, + "inactive_split_bytes_current_delta": -937426944, + "inactive_split_bytes_peak": 14421322240, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T142153627484Z.pickle", + "timestamp_utc": "2026-08-01T14:21:53.729545Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T142153627484Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T142153627484Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e86b3a37ee1d6e1f402db112b61d989677a6e2f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T142153627484Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52ed3e4d47f01c19aeb70b360169da41202ac7ce0586ac1d1f33f40e012c3a47 +size 6662823 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T142250289477Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T142250289477Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34ddd270d9e58ef167a8a0e7ebb80f8d4fb3e798 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T142250289477Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166810624, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113165513216, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166810624, + "active_bytes_peak": 113165513216, + "allocated_bytes_current": 61166810624, + "allocated_bytes_peak": 113165513216, + "device": 7, + "inactive_split_bytes_current": 9930836480, + "inactive_split_bytes_peak": 14441505280, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:22:50.289013Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166810624, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113165513216, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 113165513216, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 113165513216, + "device": 7, + "inactive_split_bytes_current": 8588659200, + "inactive_split_bytes_peak": 14421322240, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 116907835392, + "reserved_bytes_peak": 116907835392, + "segment_count_current": 474, + "segment_count_peak": 474, + "timestamp_utc": "2026-08-01T14:21:53.742741Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9930836480, + "inactive_split_bytes_current_delta": 1342177280, + "inactive_split_bytes_peak": 14441505280, + "inactive_split_bytes_peak_delta": 20183040, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 116907835392, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 116907835392, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 474, + "segment_count_current_delta": 0, + "segment_count_peak": 474, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T142250289477Z.pickle", + "timestamp_utc": "2026-08-01T14:22:50.500376Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T142250289477Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T142250289477Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..040424a7972fee1673362f2d20aa29bb890d11d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T142250289477Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67953ee3b38b37a2741154758c75de7287f6cc71036f438441f7dce04961ec42 +size 9835033 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..ba74883efd6bc42aa323af4b235091b7c3f09a7e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 4096 +capacity: 1.25 +expert_dispatch_overlap: on +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6042 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 221 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T142536998105Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T142536998105Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0bda0cbf279b6619c8f1ff0e923d0b0471e94fd5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T142536998105Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 89627064320, + "active_bytes_peak_delta": 54160728064, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89627064320, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89627064320, + "device": 0, + "inactive_split_bytes_current": 8738294784, + "inactive_split_bytes_peak": 12204020736, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98081701888, + "reserved_bytes_peak": 98081701888, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:25:36.997711Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 89627064320, + "allocated_bytes_peak_delta": 54160728064, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T14:24:29.931133Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8738294784, + "inactive_split_bytes_current_delta": 8603378688, + "inactive_split_bytes_peak": 12204020736, + "inactive_split_bytes_peak_delta": 12043388928, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98081701888, + "reserved_bytes_current_delta": 58835599360, + "reserved_bytes_peak": 98081701888, + "reserved_bytes_peak_delta": 58835599360, + "segment_count_current": 463, + "segment_count_current_delta": 391, + "segment_count_peak": 463, + "segment_count_peak_delta": 391, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T142536998105Z.pickle", + "timestamp_utc": "2026-08-01T14:25:37.107822Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T142536998105Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T142536998105Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27b53d2775d68b12944a3d469ebb2b4cb6797b6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T142536998105Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fefbff6a58af06d7c93ec5567c5714311aaf7dd9457626eea0314f4249274837 +size 5486044 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T142538188654Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T142538188654Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e8d51007806b21ad3bf3744f8ef61968c277c4c1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T142538188654Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113254822912, + "active_bytes_peak_delta": 23627758592, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113254822912, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113254822912, + "device": 0, + "inactive_split_bytes_current": 8285309952, + "inactive_split_bytes_peak": 12204020736, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:38.188250Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113254822912, + "allocated_bytes_peak_delta": 23627758592, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89627064320, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89627064320, + "device": 0, + "inactive_split_bytes_current": 8738294784, + "inactive_split_bytes_peak": 12204020736, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98081701888, + "reserved_bytes_peak": 98081701888, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:25:37.127486Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8285309952, + "inactive_split_bytes_current_delta": -452984832, + "inactive_split_bytes_peak": 12204020736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_current_delta": 19144900608, + "reserved_bytes_peak": 117226602496, + "reserved_bytes_peak_delta": 19144900608, + "segment_count_current": 525, + "segment_count_current_delta": 62, + "segment_count_peak": 525, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T142538188654Z.pickle", + "timestamp_utc": "2026-08-01T14:25:38.300534Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T142538188654Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T142538188654Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d663750651d1b5dfc4cf2433ca837812dddeae52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T142538188654Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a19d7bb06cf4343a6d7e152a6000f100bcd958db37a7621433bd9eb17f5400ac +size 5615523 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T142539355270Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T142539355270Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7267495df7d349abbb4528caac9918d9765c3887 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T142539355270Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259319808, + "active_bytes_peak_delta": 4496896, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259319808, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259319808, + "device": 0, + "inactive_split_bytes_current": 8283212800, + "inactive_split_bytes_peak": 12204020736, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:39.354870Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259319808, + "allocated_bytes_peak_delta": 4496896, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113254822912, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113254822912, + "device": 0, + "inactive_split_bytes_current": 8285309952, + "inactive_split_bytes_peak": 12204020736, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:38.315860Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8283212800, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 12204020736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117226602496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T142539355270Z.pickle", + "timestamp_utc": "2026-08-01T14:25:39.460966Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T142539355270Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T142539355270Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cb6347b131bb9929416313594d0373655a99333b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T142539355270Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e2cd574a5f1fcaf2997d4b0948c76e332852da06a6e333f467a5653f7084c4d +size 5615514 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T142540525962Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T142540525962Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7ccee0ffa9484393fe9b9338f3831d5274dda1a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T142540525962Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259319808, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259319808, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259319808, + "device": 0, + "inactive_split_bytes_current": 8274824192, + "inactive_split_bytes_peak": 12204020736, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:40.525556Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259319808, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259319808, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259319808, + "device": 0, + "inactive_split_bytes_current": 8283212800, + "inactive_split_bytes_peak": 12204020736, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:39.475967Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8274824192, + "inactive_split_bytes_current_delta": -8388608, + "inactive_split_bytes_peak": 12204020736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117226602496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T142540525962Z.pickle", + "timestamp_utc": "2026-08-01T14:25:40.630147Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T142540525962Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T142540525962Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6111bad9398d27c436da8dbacc847e7ec4b3d99 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T142540525962Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a79627ffa2dd2c1ddc185819e0de0f9389979813c7b4fd5da480f6ac481dde3e +size 5615245 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T142541969910Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T142541969910Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a27d45b86d28cd8b26598bbc0d0f1433ac5a058 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T142541969910Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113262072320, + "active_bytes_peak_delta": 2752512, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262072320, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262072320, + "device": 0, + "inactive_split_bytes_current": 8276921344, + "inactive_split_bytes_peak": 12204020736, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:41.969485Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113262072320, + "allocated_bytes_peak_delta": 2752512, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259319808, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259319808, + "device": 0, + "inactive_split_bytes_current": 8274824192, + "inactive_split_bytes_peak": 12204020736, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:40.644994Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8276921344, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12204020736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117226602496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T142541969910Z.pickle", + "timestamp_utc": "2026-08-01T14:25:42.074028Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T142541969910Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T142541969910Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6a38b1923ee4dd817cf3dc4ac852c0940ec3102 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T142541969910Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb12a0ca86a46a954d426ef3034baee2dab4a63aa77e2c7163acac60c2f9a4b4 +size 5618296 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T142543143452Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T142543143452Z.json new file mode 100644 index 0000000000000000000000000000000000000000..32fe72af777761773cb98c64b6b00708551734de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T142543143452Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113262072320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262072320, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262072320, + "device": 0, + "inactive_split_bytes_current": 8272727040, + "inactive_split_bytes_peak": 12204020736, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:43.142972Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113262072320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262072320, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262072320, + "device": 0, + "inactive_split_bytes_current": 8276921344, + "inactive_split_bytes_peak": 12204020736, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:42.088607Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8272727040, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 12204020736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117226602496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T142543143452Z.pickle", + "timestamp_utc": "2026-08-01T14:25:43.245434Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T142543143452Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T142543143452Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9bfc52f5ef1bfba71be9b0eb1a52b51a52310654 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T142543143452Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c6abf4fa197d4ff91bd4b1fc550a907c85bd2b9ad60bbe4d7a116c88593c515 +size 5615038 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T142544301794Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T142544301794Z.json new file mode 100644 index 0000000000000000000000000000000000000000..81a8536a3a0db1514378cc3e3c896f277e4ff071 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T142544301794Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113262072320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262072320, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262072320, + "device": 0, + "inactive_split_bytes_current": 8283212800, + "inactive_split_bytes_peak": 12204020736, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:44.301397Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113262072320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262072320, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262072320, + "device": 0, + "inactive_split_bytes_current": 8272727040, + "inactive_split_bytes_peak": 12204020736, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:43.259934Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8283212800, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 12204020736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117226602496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T142544301794Z.pickle", + "timestamp_utc": "2026-08-01T14:25:44.401961Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T142544301794Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T142544301794Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2beb7787cabd1e20cd614ee59805d28a88bc9069 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T142544301794Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65ac6a7cb4f46c6ed0bf9e7ba840e7685a0a67820607d9b46c9af821c896b5b3 +size 5615307 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T142545460251Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T142545460251Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1a4a894ded6e562b512fa57be5e5c110f4f9e374 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T142545460251Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113262072320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262072320, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262072320, + "device": 0, + "inactive_split_bytes_current": 8272727040, + "inactive_split_bytes_peak": 12204020736, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:45.459831Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113262072320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262072320, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262072320, + "device": 0, + "inactive_split_bytes_current": 8283212800, + "inactive_split_bytes_peak": 12204020736, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:44.416297Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8272727040, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 12204020736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117226602496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T142545460251Z.pickle", + "timestamp_utc": "2026-08-01T14:25:45.561817Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T142545460251Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T142545460251Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..70e76d01093ea51fb31b4b2ed3fb695d312384e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T142545460251Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d7e2245a406298d00c71d1499fda8ed13c9d3f5f9fc1260517f52760dac9121 +size 5660289 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T142546616629Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T142546616629Z.json new file mode 100644 index 0000000000000000000000000000000000000000..90e03cf4ed9b208de904e0d8081e19d11fb27f0d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T142546616629Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113262072320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262072320, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262072320, + "device": 0, + "inactive_split_bytes_current": 8276921344, + "inactive_split_bytes_peak": 12204020736, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:46.616240Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113262072320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262072320, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262072320, + "device": 0, + "inactive_split_bytes_current": 8272727040, + "inactive_split_bytes_peak": 12204020736, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:45.576418Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8276921344, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 12204020736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117226602496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T142546616629Z.pickle", + "timestamp_utc": "2026-08-01T14:25:46.715706Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T142546616629Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T142546616629Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d38ec38aecebb0c5a693c94cb4a78b85c0d0ac42 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T142546616629Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6ca14c19e6e944ac160fba01ae99c5be12ded4965e6cb21a15ab997dbdb4a64 +size 5618187 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T142639862895Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T142639862895Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e4c2d2b8cb1289bb076498be0f0695dff4f1f9ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T142639862895Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113262072320, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 113262072320, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 113262072320, + "device": 0, + "inactive_split_bytes_current": 8004602880, + "inactive_split_bytes_peak": 12204020736, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:26:39.862422Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113262072320, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113262072320, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113262072320, + "device": 0, + "inactive_split_bytes_current": 8276921344, + "inactive_split_bytes_peak": 12204020736, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_peak": 117226602496, + "segment_count_current": 525, + "segment_count_peak": 525, + "timestamp_utc": "2026-08-01T14:25:46.729904Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8004602880, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 12204020736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 117226602496, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117226602496, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 525, + "segment_count_current_delta": 0, + "segment_count_peak": 525, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T142639862895Z.pickle", + "timestamp_utc": "2026-08-01T14:26:40.058432Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T142639862895Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T142639862895Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b51aca7155478cd6431e3029d3ecb3eaf7d9382 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T142639862895Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fec843daeffad68b78032362e8bfef01f4b77284c11840c7299316dc2ff3901 +size 9430324 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T142536998106Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T142536998106Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0b5437ad5bdfb09225ca81f79748d68ed082f751 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T142536998106Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 89621532672, + "active_bytes_peak_delta": 54155196416, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89621532672, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89621532672, + "device": 1, + "inactive_split_bytes_current": 8599882752, + "inactive_split_bytes_peak": 10716771328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 98314485760, + "reserved_bytes_peak": 98314485760, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:25:36.997547Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 89621532672, + "allocated_bytes_peak_delta": 54155196416, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:24:29.931615Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8599882752, + "inactive_split_bytes_current_delta": 8467063808, + "inactive_split_bytes_peak": 10716771328, + "inactive_split_bytes_peak_delta": 10556385280, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 98314485760, + "reserved_bytes_current_delta": 59070480384, + "reserved_bytes_peak": 98314485760, + "reserved_bytes_peak_delta": 59070480384, + "segment_count_current": 463, + "segment_count_current_delta": 392, + "segment_count_peak": 463, + "segment_count_peak_delta": 392, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T142536998106Z.pickle", + "timestamp_utc": "2026-08-01T14:25:37.107201Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T142536998106Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T142536998106Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c6cda63c04d7e73ea403e50a8a0baae4734eff65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T142536998106Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fad67d014ce139f9681a18567a05210641e42f914a56b2c686a1ca8fdb68580 +size 5395413 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T142538188663Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T142538188663Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bd9d8f1a6088d268f4c5277b04e5ce005772d5e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T142538188663Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259712512, + "active_bytes_peak_delta": 23638179840, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259712512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259712512, + "device": 1, + "inactive_split_bytes_current": 8381778944, + "inactive_split_bytes_peak": 10909739008, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:38.188257Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259712512, + "allocated_bytes_peak_delta": 23638179840, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89621532672, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89621532672, + "device": 1, + "inactive_split_bytes_current": 8599882752, + "inactive_split_bytes_peak": 10716771328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 98314485760, + "reserved_bytes_peak": 98314485760, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:25:37.124911Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8381778944, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 10909739008, + "inactive_split_bytes_peak_delta": 192967680, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 18903728128, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 18903728128, + "segment_count_current": 523, + "segment_count_current_delta": 60, + "segment_count_peak": 523, + "segment_count_peak_delta": 60, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T142538188663Z.pickle", + "timestamp_utc": "2026-08-01T14:25:38.297944Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T142538188663Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T142538188663Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..70a3fa70605942ced26f2c426c265f2808421281 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T142538188663Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c33ec206878053a1e01a24b5d4eeebdce5894525d299bdb155d0bf4d65e9f43 +size 5580324 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T142539355299Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T142539355299Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1553e0bd9230b2a122e86a59775f31cc55b9f0c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T142539355299Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975168, + "active_bytes_peak_delta": 262656, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 11898851328, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:39.354900Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975168, + "allocated_bytes_peak_delta": 262656, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259712512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259712512, + "device": 1, + "inactive_split_bytes_current": 8381778944, + "inactive_split_bytes_peak": 10909739008, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:38.311308Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11898851328, + "inactive_split_bytes_peak_delta": 989112320, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T142539355299Z.pickle", + "timestamp_utc": "2026-08-01T14:25:39.455147Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T142539355299Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T142539355299Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b8288e533a2f0c3686c9a82422f2ba9cb8535a92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T142539355299Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b21ce3a28a33c44bf98d244c7eff2a6996bd71593fded2d44a89ab7efb34fe4 +size 5589439 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T142540526019Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T142540526019Z.json new file mode 100644 index 0000000000000000000000000000000000000000..947d1ae4b13c3cddabf488c29b3e1bad6f610ade --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T142540526019Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8371293184, + "inactive_split_bytes_peak": 11898851328, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:40.525609Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 11898851328, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:39.468207Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8371293184, + "inactive_split_bytes_current_delta": -8388608, + "inactive_split_bytes_peak": 11898851328, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T142540526019Z.pickle", + "timestamp_utc": "2026-08-01T14:25:40.627818Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T142540526019Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T142540526019Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6939b2f161a763f71faaeffa6d7132152ccd618e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T142540526019Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66b5adbc6e34c425dc0e96f65927ed94a5860f4e76a724e51a1455fd0e4cdcfd +size 5597795 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T142541969909Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T142541969909Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a2ebd1672d90251df0f742c8de2df8ebd40396aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T142541969909Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 12167680000, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:41.969501Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8371293184, + "inactive_split_bytes_peak": 11898851328, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:40.640716Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12167680000, + "inactive_split_bytes_peak_delta": 268828672, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T142541969909Z.pickle", + "timestamp_utc": "2026-08-01T14:25:42.073640Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T142541969909Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T142541969909Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3a56e5f97a7d6dd32e7904cdb77d765725eda695 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T142541969909Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce28ac4001570394cec9cd0014b696fe9049aa2a4017bb63a41e4323fa1c4460 +size 5582417 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T142543143414Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T142543143414Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2c637a2ab957fb540d432d7ae0eaf731bc97b55b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T142543143414Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 12167680000, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:43.142989Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 12167680000, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:42.086683Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 12167680000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T142543143414Z.pickle", + "timestamp_utc": "2026-08-01T14:25:43.241075Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T142543143414Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T142543143414Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9cd0bc2fb445d589d2d56393e7e1f727ef8b3c00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T142543143414Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a283ae8f0bf9a5c4e7d1b82dab384773d075c55fd58d2af7b4880b3a06a944e3 +size 5618550 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T142544301803Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T142544301803Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c171fdb220144f9905b82d8d1f1a25bf822a1560 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T142544301803Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 12167680000, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:44.301412Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 12167680000, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:43.253633Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 12167680000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T142544301803Z.pickle", + "timestamp_utc": "2026-08-01T14:25:44.401750Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T142544301803Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T142544301803Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e29d3e855d457662324739a813b47009a9525cf2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T142544301803Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78cb5fce7a7dfc2afc67720507bf794b9e44b11c92b2a8ceedd3a264ee8e3721 +size 5589281 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T142545460256Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T142545460256Z.json new file mode 100644 index 0000000000000000000000000000000000000000..68f2636bd576b00a9714e9e29a877bc1ae5f2bcd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T142545460256Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 12167680000, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:45.459833Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 12167680000, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:44.416531Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 12167680000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T142545460256Z.pickle", + "timestamp_utc": "2026-08-01T14:25:45.555772Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T142545460256Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T142545460256Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0b858ca5617067d7dbe4e7ad44bb0aef34deab2c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T142545460256Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f5dbd81ed9bebb9e54b074a52b2249ec4ae17ad03c473683529b11321774cdd +size 5621686 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T142546616678Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T142546616678Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cef4ecb66606c3108351bf1c8084442b1dea5a3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T142546616678Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8608271360, + "inactive_split_bytes_peak": 12167680000, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:46.616264Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 12167680000, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:45.568140Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8608271360, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 12167680000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T142546616678Z.pickle", + "timestamp_utc": "2026-08-01T14:25:46.711861Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T142546616678Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T142546616678Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df3fdc870192c9aabdbbf55697cfa540c0b7d211 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T142546616678Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7791cb52ead0baf860a72ff20eb9ce4112a9a486ba90bd88682361c68a1d4dc +size 5582448 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T142639862915Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T142639862915Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8f710a02d3c82ab2b9e6b76dbdec958b2698588b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T142639862915Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113259975168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8101071872, + "inactive_split_bytes_peak": 12167680000, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:26:39.862429Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113259975168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 1, + "inactive_split_bytes_current": 8608271360, + "inactive_split_bytes_peak": 12167680000, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:46.724195Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8101071872, + "inactive_split_bytes_current_delta": -507199488, + "inactive_split_bytes_peak": 12167680000, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T142639862915Z.pickle", + "timestamp_utc": "2026-08-01T14:26:40.058256Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T142639862915Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T142639862915Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..eb40b23df951640a77065b8ef495331f19a8506e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T142639862915Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f79d340fde460592ed973ba198673de6ce27f0fb3224ea0abd660e913c1b806 +size 9381896 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T142536998104Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T142536998104Z.json new file mode 100644 index 0000000000000000000000000000000000000000..54f9efdee589274353ff64eab197b185eb07122b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T142536998104Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 89621532672, + "active_bytes_peak_delta": 54155196416, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89621532672, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89621532672, + "device": 2, + "inactive_split_bytes_current": 8599882752, + "inactive_split_bytes_peak": 10716771328, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98314485760, + "reserved_bytes_peak": 98314485760, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:25:36.997553Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 89621532672, + "allocated_bytes_peak_delta": 54155196416, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:24:29.931623Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8599882752, + "inactive_split_bytes_current_delta": 8467063808, + "inactive_split_bytes_peak": 10716771328, + "inactive_split_bytes_peak_delta": 10556385280, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98314485760, + "reserved_bytes_current_delta": 59070480384, + "reserved_bytes_peak": 98314485760, + "reserved_bytes_peak_delta": 59070480384, + "segment_count_current": 463, + "segment_count_current_delta": 392, + "segment_count_peak": 463, + "segment_count_peak_delta": 392, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T142536998104Z.pickle", + "timestamp_utc": "2026-08-01T14:25:37.106172Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T142536998104Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T142536998104Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..18c968a37dfde78bfe1f6a3da5d7a8296063ab7c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T142536998104Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3831e3fab9c0b84c69d8b99acd16acd0c41b6fa0a39f11755ffd3e7e973fcba8 +size 5395402 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T142538188645Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T142538188645Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b265d863e6f376d83dcf83edb8d0f178031c6e3f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T142538188645Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113254731776, + "active_bytes_peak_delta": 23633199104, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113254731776, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113254731776, + "device": 2, + "inactive_split_bytes_current": 8381778944, + "inactive_split_bytes_peak": 11650455552, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:38.188236Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113254731776, + "allocated_bytes_peak_delta": 23633199104, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89621532672, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89621532672, + "device": 2, + "inactive_split_bytes_current": 8599882752, + "inactive_split_bytes_peak": 10716771328, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98314485760, + "reserved_bytes_peak": 98314485760, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:25:37.123543Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8381778944, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 11650455552, + "inactive_split_bytes_peak_delta": 933684224, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117365014528, + "reserved_bytes_current_delta": 19050528768, + "reserved_bytes_peak": 117365014528, + "reserved_bytes_peak_delta": 19050528768, + "segment_count_current": 522, + "segment_count_current_delta": 59, + "segment_count_peak": 522, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T142538188645Z.pickle", + "timestamp_utc": "2026-08-01T14:25:38.297348Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T142538188645Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T142538188645Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bc5d2222a03aae3d597abcb8a7745042c9810f91 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T142538188645Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:378294b03e94ed73dcbb8861f981b6cfe4882eb1704e40f184710623d0103a74 +size 5579634 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T142539355289Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T142539355289Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2d7de3f2805c4ea1106166e686a87cfeee5290f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T142539355289Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260106240, + "active_bytes_peak_delta": 5374464, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 11650455552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:39.354887Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260106240, + "allocated_bytes_peak_delta": 5374464, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113254731776, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113254731776, + "device": 2, + "inactive_split_bytes_current": 8381778944, + "inactive_split_bytes_peak": 11650455552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:38.310777Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11650455552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117365014528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117365014528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T142539355289Z.pickle", + "timestamp_utc": "2026-08-01T14:25:39.459580Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T142539355289Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T142539355289Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ac23d9f565e5515eeafd5b106eb77dd6896ff5e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T142539355289Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f05cc0411fb242ff2a5e72504888a3174c5099852c87158f77966712d441a25 +size 5589020 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T142540525968Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T142540525968Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5da5713345f4d95501b57b89e3a3ab2de30721b3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T142540525968Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260106240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8371293184, + "inactive_split_bytes_peak": 11650455552, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:40.525560Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260106240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 11650455552, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:39.472823Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8371293184, + "inactive_split_bytes_current_delta": -8388608, + "inactive_split_bytes_peak": 11650455552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117365014528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117365014528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T142540525968Z.pickle", + "timestamp_utc": "2026-08-01T14:25:40.627304Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T142540525968Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T142540525968Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..405dd1c8af3b8df3c427a18a2dbe0cd5e21bb5d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T142540525968Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86cd2cf0abe270058a1c35ec09b4972bae71a5f38e662a94f85a8c6986acc4e6 +size 5606484 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T142541969892Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T142541969892Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c77241a8a3278e688e459236ff0c3e2acd9ff765 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T142541969892Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260106240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 11650455552, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:41.969490Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260106240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8371293184, + "inactive_split_bytes_peak": 11650455552, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:40.640378Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11650455552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117365014528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117365014528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T142541969892Z.pickle", + "timestamp_utc": "2026-08-01T14:25:42.074286Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T142541969892Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T142541969892Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d29bcfc684801957542c5df667c1964f8c63bbe0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T142541969892Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83a304569789df2b27bd8b52ecc04227c06c94023fb07b3933a7051d0cbe948d +size 5586478 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T142543143372Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T142543143372Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a27634481bde89aa3f6d4f32d8bc367efeb0cdf5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T142543143372Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260106240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 11650455552, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:43.142962Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260106240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 11650455552, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:42.087919Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 11650455552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117365014528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117365014528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T142543143372Z.pickle", + "timestamp_utc": "2026-08-01T14:25:43.243260Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T142543143372Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T142543143372Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..370b82a3264d87be4747a1adf40214a57ccc13c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T142543143372Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27386821b91e8829d8c9da21216b550885115e33161665ec61a27f754f3673bd +size 5621307 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T142544301804Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T142544301804Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67275fd8f074050f29246c0507be8aa694dc5992 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T142544301804Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260106240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 11650455552, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:44.301397Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260106240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 11650455552, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:43.255912Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 11650455552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117365014528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117365014528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T142544301804Z.pickle", + "timestamp_utc": "2026-08-01T14:25:44.401337Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T142544301804Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T142544301804Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aa56f00c7e0035c47994dbfcb1a1c79c39e480e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T142544301804Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b08a9f4c009b5e2ebce4b2fd816192b2dc09fb4173c55eb7160fce8f5d013a5 +size 5588896 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T142545460281Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T142545460281Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4c9d05e2570480f3936d2f98239b3a580ec540bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T142545460281Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260106240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 11650455552, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:45.459801Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260106240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 11650455552, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:44.413960Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 11650455552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117365014528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117365014528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T142545460281Z.pickle", + "timestamp_utc": "2026-08-01T14:25:45.560222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T142545460281Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T142545460281Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0cca9a8cdbef988c5037cfbe85b90d09a9b7df74 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T142545460281Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ecfe47fe1e19fbda47b93bf35d60dc3afbaf570d88e661dcef8d39f50bee5e2 +size 5621238 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T142546616616Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T142546616616Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0a12b084eaa1f3fd7ecd27f03029c5847fca461f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T142546616616Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260106240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 11650455552, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:46.616220Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260106240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 11650455552, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:45.572786Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 11650455552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117365014528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117365014528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T142546616616Z.pickle", + "timestamp_utc": "2026-08-01T14:25:46.715068Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T142546616616Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T142546616616Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3c316c1f2f5bc1a7d55ff5c0b33d62d0e3285245 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T142546616616Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c2649bef7cd0a088a95baec18e76a33c318726a17f2011356a0c852846624c5 +size 5586425 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T142639862911Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T142639862911Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42401daeeaab4ab9cb53c80ee57b66b880c19fc2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T142639862911Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113260106240, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8101071872, + "inactive_split_bytes_peak": 11650455552, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:26:39.862436Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113260106240, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113260106240, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113260106240, + "device": 2, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 11650455552, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117365014528, + "reserved_bytes_peak": 117365014528, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:46.727649Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8101071872, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 11650455552, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117365014528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117365014528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T142639862911Z.pickle", + "timestamp_utc": "2026-08-01T14:26:40.060447Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T142639862911Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T142639862911Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a14adb5cc8d0abaaec7c401841d7bad6de9c7609 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T142639862911Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22a5c54aa3ba4befd5bbf9d4617ccca31f223c47f576f76367bfc66ef28b65e2 +size 9381416 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T142536998102Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T142536998102Z.json new file mode 100644 index 0000000000000000000000000000000000000000..999f8b6e54f907f8922749840f945ed71e570d3f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T142536998102Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 89621532672, + "active_bytes_peak_delta": 54155196416, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89621532672, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89621532672, + "device": 3, + "inactive_split_bytes_current": 8599882752, + "inactive_split_bytes_peak": 10437268992, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 98314485760, + "reserved_bytes_peak": 98314485760, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:25:36.997532Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 89621532672, + "allocated_bytes_peak_delta": 54155196416, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:24:29.932068Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8599882752, + "inactive_split_bytes_current_delta": 8467063808, + "inactive_split_bytes_peak": 10437268992, + "inactive_split_bytes_peak_delta": 10276882944, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 98314485760, + "reserved_bytes_current_delta": 59070480384, + "reserved_bytes_peak": 98314485760, + "reserved_bytes_peak_delta": 59070480384, + "segment_count_current": 463, + "segment_count_current_delta": 392, + "segment_count_peak": 463, + "segment_count_peak_delta": 392, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T142536998102Z.pickle", + "timestamp_utc": "2026-08-01T14:25:37.107905Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T142536998102Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T142536998102Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a46121d457cf66d8afc6571cc9bd7e893e9d38c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T142536998102Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3cd761013d638bedd8faa99febc14ed14313d6f11c057d93372a0b94bcff315 +size 5395400 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T142538188663Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T142538188663Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2b98aaa61cfb9f738ac6810a3cf78ff2ec75b2cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T142538188663Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259712512, + "active_bytes_peak_delta": 23638179840, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259712512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259712512, + "device": 3, + "inactive_split_bytes_current": 8381778944, + "inactive_split_bytes_peak": 10905544704, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:38.188250Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259712512, + "allocated_bytes_peak_delta": 23638179840, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89621532672, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89621532672, + "device": 3, + "inactive_split_bytes_current": 8599882752, + "inactive_split_bytes_peak": 10437268992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 98314485760, + "reserved_bytes_peak": 98314485760, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:25:37.125284Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8381778944, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 10905544704, + "inactive_split_bytes_peak_delta": 468275712, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 18903728128, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 18903728128, + "segment_count_current": 523, + "segment_count_current_delta": 60, + "segment_count_peak": 523, + "segment_count_peak_delta": 60, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T142538188663Z.pickle", + "timestamp_utc": "2026-08-01T14:25:38.297964Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T142538188663Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T142538188663Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44e207ec4829dd24e149232c793fd16fbf2cdf3a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T142538188663Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab51f996df5a4d1995527535067eef6d379c1c5c643966af1493c8b883bdb33c +size 5580178 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T142539355275Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T142539355275Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bed1871b279127716aed000cf0caf70284153e31 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T142539355275Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975168, + "active_bytes_peak_delta": 262656, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 3, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 11900948480, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:39.354876Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975168, + "allocated_bytes_peak_delta": 262656, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259712512, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259712512, + "device": 3, + "inactive_split_bytes_current": 8381778944, + "inactive_split_bytes_peak": 10905544704, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:38.311326Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11900948480, + "inactive_split_bytes_peak_delta": 995403776, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T142539355275Z.pickle", + "timestamp_utc": "2026-08-01T14:25:39.459823Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T142539355275Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T142539355275Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6283d01917686eda687b963568014e32406ffde7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T142539355275Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2748be9e05f9e38275d20a0194ae6b3c77661baa954972b6cbf2d20cb1cb50a8 +size 5589452 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T142540525969Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T142540525969Z.json new file mode 100644 index 0000000000000000000000000000000000000000..480be36556c24fc2eb916baa1f61d5c74a91ca90 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T142540525969Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 3, + "inactive_split_bytes_current": 8371293184, + "inactive_split_bytes_peak": 11900948480, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:40.525556Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 3, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 11900948480, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:39.473067Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8371293184, + "inactive_split_bytes_current_delta": -8388608, + "inactive_split_bytes_peak": 11900948480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T142540525969Z.pickle", + "timestamp_utc": "2026-08-01T14:25:40.629279Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T142540525969Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T142540525969Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7d12df19919d10456433fb6fa1c5fc5ee0c79b61 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T142540525969Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ce881763eeacc1322dd18186201d57e2086a6ef790e7e2193ce57a90badc9c5 +size 5597952 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T142541969892Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T142541969892Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34e26c7acd69bcf709ba79401004ccf68f5126ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T142541969892Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975168, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 3, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 12161388544, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:41.969485Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975168, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 3, + "inactive_split_bytes_current": 8371293184, + "inactive_split_bytes_peak": 11900948480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:40.643955Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 12161388544, + "inactive_split_bytes_peak_delta": 260440064, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T142541969892Z.pickle", + "timestamp_utc": "2026-08-01T14:25:42.075437Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T142541969892Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T142541969892Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a6991073940eb93b5001c606c7bedc219650a9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T142541969892Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73f8ac41568d56d584f3e03217112c0733db86252079414503964497ea89974e +size 5582532 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T142543143415Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T142543143415Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70af2c4445e31c1eabbd0057e19777a2e859e7ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T142543143415Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261023744, + "active_bytes_peak_delta": 1048576, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261023744, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261023744, + "device": 3, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 12161388544, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:43.143016Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261023744, + "allocated_bytes_peak_delta": 1048576, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975168, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975168, + "device": 3, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 12161388544, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:42.089196Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 12161388544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T142543143415Z.pickle", + "timestamp_utc": "2026-08-01T14:25:43.243886Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T142543143415Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T142543143415Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..16df2d1dd86efa0015308ff330f51d56250539fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T142543143415Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23cc2d07be4ba3cff5c592e7af1788ec0efaba988202ff9476cd89de46d3d660 +size 5618498 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T142544301804Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T142544301804Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4eb901a21c05da8f792958b54a45a014aa381a6c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T142544301804Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261023744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261023744, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261023744, + "device": 3, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 12161388544, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:44.301402Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261023744, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261023744, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261023744, + "device": 3, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 12161388544, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:43.262117Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 12161388544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T142544301804Z.pickle", + "timestamp_utc": "2026-08-01T14:25:44.407738Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T142544301804Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T142544301804Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8247b048dd61782f6bd9d6610688445b0ed8f0ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T142544301804Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:760827394e0a03eb573b55be81d8faa66e6045ece16028b0a00f57a7b50a6fef +size 5589263 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T142545460245Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T142545460245Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db7ed5c45be5096afd18e08daf194800295f0e8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T142545460245Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261023744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261023744, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261023744, + "device": 3, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 12161388544, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:45.459817Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261023744, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261023744, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261023744, + "device": 3, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 12161388544, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:44.423349Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 12161388544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T142545460245Z.pickle", + "timestamp_utc": "2026-08-01T14:25:45.560852Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T142545460245Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T142545460245Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7736c6d5e6f8c77d5127de480c371f92d95e72be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T142545460245Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1d247d9fa101583658bf35207a1d5f9ab30d2d5a5da31ed7682106360127e33 +size 5621608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T142546616654Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T142546616654Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d5a0d508e774fa202a3879574fe1a133280e98a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T142546616654Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261023744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261023744, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261023744, + "device": 3, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 12161388544, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:46.616255Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261023744, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261023744, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261023744, + "device": 3, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 12161388544, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:45.573522Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 12161388544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T142546616654Z.pickle", + "timestamp_utc": "2026-08-01T14:25:46.717523Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T142546616654Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T142546616654Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d4e62dcbc6b487a5a08d46a121ae1e67fbee095c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T142546616654Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11ca1d4311ee84e344730580d749b220db326e15a5a8f1671648859981d70bac +size 5582256 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T142639862913Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T142639862913Z.json new file mode 100644 index 0000000000000000000000000000000000000000..438e3e5515841928fa8126798eec1f80cf23ff10 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T142639862913Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113261023744, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 113261023744, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 113261023744, + "device": 3, + "inactive_split_bytes_current": 8101071872, + "inactive_split_bytes_peak": 12161388544, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:26:39.862439Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113261023744, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261023744, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261023744, + "device": 3, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 12161388544, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:46.730373Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8101071872, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 12161388544, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T142639862913Z.pickle", + "timestamp_utc": "2026-08-01T14:26:40.061269Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T142639862913Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T142639862913Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a31339b2567a397007353ac281ab8d2c3b209b22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T142639862913Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5751eb74115bbf2233add26d159090a5806549aa0ba79391320acd706bc0ffc +size 9381657 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T142536998154Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T142536998154Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4a7900f3a6bdac2ce6eec852a541ca31ea94cda3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T142536998154Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855123968, + "active_bytes_current_delta": 31623226368, + "active_bytes_peak": 89622450176, + "active_bytes_peak_delta": 54155671552, + "after": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 89622450176, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 89622450176, + "device": 4, + "inactive_split_bytes_current": 8495547392, + "inactive_split_bytes_peak": 10716938240, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 98608087040, + "reserved_bytes_peak": 98608087040, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T14:25:36.997736Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855123968, + "allocated_bytes_current_delta": 31623226368, + "allocated_bytes_peak": 89622450176, + "allocated_bytes_peak_delta": 54155671552, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:24:29.931890Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8495547392, + "inactive_split_bytes_current_delta": 8363170816, + "inactive_split_bytes_peak": 10716938240, + "inactive_split_bytes_peak_delta": 10563528704, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 98608087040, + "reserved_bytes_current_delta": 59364081664, + "reserved_bytes_peak": 98608087040, + "reserved_bytes_peak_delta": 59364081664, + "segment_count_current": 467, + "segment_count_current_delta": 396, + "segment_count_peak": 467, + "segment_count_peak_delta": 396, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T142536998154Z.pickle", + "timestamp_utc": "2026-08-01T14:25:37.110063Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T142536998154Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T142536998154Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..422c28b3acaead2cc8147b1d41c818e1d87adecd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T142536998154Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d3f48ae49eac9d7a91e23508f6f54aaf2f1af328fe36e513125c7f51d864612 +size 5485238 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T142538188664Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T142538188664Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fd506639eb42f39ae123f76b58ba844ae88451cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T142538188664Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855123968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113251718144, + "active_bytes_peak_delta": 23629267968, + "after": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113251718144, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113251718144, + "device": 4, + "inactive_split_bytes_current": 8277443584, + "inactive_split_bytes_peak": 11918432256, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:38.188269Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855123968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113251718144, + "allocated_bytes_peak_delta": 23629267968, + "before": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 89622450176, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 89622450176, + "device": 4, + "inactive_split_bytes_current": 8495547392, + "inactive_split_bytes_peak": 10716938240, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 98608087040, + "reserved_bytes_peak": 98608087040, + "segment_count_current": 467, + "segment_count_peak": 467, + "timestamp_utc": "2026-08-01T14:25:37.127624Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8277443584, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 11918432256, + "inactive_split_bytes_peak_delta": 1201494016, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117190950912, + "reserved_bytes_current_delta": 18582863872, + "reserved_bytes_peak": 117190950912, + "reserved_bytes_peak_delta": 18582863872, + "segment_count_current": 524, + "segment_count_current_delta": 57, + "segment_count_peak": 524, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T142538188664Z.pickle", + "timestamp_utc": "2026-08-01T14:25:38.298127Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T142538188664Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T142538188664Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a3288b907e403b418d98fb3a270b853dea9f859f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T142538188664Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ebb0ddd8ccaedddd6631e8d08464db7299a00184442633a0fd4e89bb5f0762b +size 5611740 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T142539355315Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T142539355315Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d31f0203ff1e92fd8b5f5f2021add28932066c06 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T142539355315Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855123968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113255191552, + "active_bytes_peak_delta": 3473408, + "after": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113255191552, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113255191552, + "device": 4, + "inactive_split_bytes_current": 8275346432, + "inactive_split_bytes_peak": 11918432256, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:39.354895Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855123968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113255191552, + "allocated_bytes_peak_delta": 3473408, + "before": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113251718144, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113251718144, + "device": 4, + "inactive_split_bytes_current": 8277443584, + "inactive_split_bytes_peak": 11918432256, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:38.312603Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8275346432, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11918432256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117190950912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117190950912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 524, + "segment_count_current_delta": 0, + "segment_count_peak": 524, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T142539355315Z.pickle", + "timestamp_utc": "2026-08-01T14:25:39.453584Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T142539355315Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T142539355315Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3de5f28c28986c5effea507fbbef32ddc00edd64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T142539355315Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1928aefa959e67199afb4d7618c5fdb7eb9bc784d68b440a436cc89cfec910b5 +size 5614359 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T142540526035Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T142540526035Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1428fbb3f26d26ac0ba55dbe738fd81d0c4aea1d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T142540526035Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855123968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113255191552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113255191552, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113255191552, + "device": 4, + "inactive_split_bytes_current": 8266957824, + "inactive_split_bytes_peak": 11918432256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:40.525607Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855123968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113255191552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113255191552, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113255191552, + "device": 4, + "inactive_split_bytes_current": 8275346432, + "inactive_split_bytes_peak": 11918432256, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:39.465557Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8266957824, + "inactive_split_bytes_current_delta": -8388608, + "inactive_split_bytes_peak": 11918432256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117190950912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117190950912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 524, + "segment_count_current_delta": 0, + "segment_count_peak": 524, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T142540526035Z.pickle", + "timestamp_utc": "2026-08-01T14:25:40.624702Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T142540526035Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T142540526035Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e2416ff53ec41b2709d87f24e3933213ea76d0b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T142540526035Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e524ae05d9dfd60cd3724464284e5f5197e92809eeef1811dd9c29b5de788ae6 +size 5614241 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T142541969940Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T142541969940Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ebbf9c5eab140edc7a6fdfb558ec6aad702f2b71 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T142541969940Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855123968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113255191552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113255191552, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113255191552, + "device": 4, + "inactive_split_bytes_current": 8269054976, + "inactive_split_bytes_peak": 11918432256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:41.969518Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855123968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113255191552, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113255191552, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113255191552, + "device": 4, + "inactive_split_bytes_current": 8266957824, + "inactive_split_bytes_peak": 11918432256, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:40.636848Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8269054976, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11918432256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117190950912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117190950912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 524, + "segment_count_current_delta": 0, + "segment_count_peak": 524, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T142541969940Z.pickle", + "timestamp_utc": "2026-08-01T14:25:42.074086Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T142541969940Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T142541969940Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..474bd6400bb484afff4761da85fd300532a18b16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T142541969940Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13b817ccf945a895c0a0fb2c46bc996255af467b355d1f4da8b67e203d9b2eef +size 5617197 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T142543143388Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T142543143388Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e13f22b36a5b1d3b69f78c410bd0e8d4bbf3ac0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T142543143388Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855123968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113256633344, + "active_bytes_peak_delta": 1441792, + "after": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113256633344, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113256633344, + "device": 4, + "inactive_split_bytes_current": 8264860672, + "inactive_split_bytes_peak": 11918432256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:43.142982Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855123968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113256633344, + "allocated_bytes_peak_delta": 1441792, + "before": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113255191552, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113255191552, + "device": 4, + "inactive_split_bytes_current": 8269054976, + "inactive_split_bytes_peak": 11918432256, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:42.088069Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8264860672, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 11918432256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117190950912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117190950912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 524, + "segment_count_current_delta": 0, + "segment_count_peak": 524, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T142543143388Z.pickle", + "timestamp_utc": "2026-08-01T14:25:43.245601Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T142543143388Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T142543143388Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9b561432c5b6965d0a45f0d7020f77f732940c8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T142543143388Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26af6926adef2ce51d62e476fd496afeaa76734e09b193acd07f0a23fb9e3283 +size 5614065 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T142544301819Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T142544301819Z.json new file mode 100644 index 0000000000000000000000000000000000000000..64b383b8d40d2ad3bbaecce2647155354774519c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T142544301819Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855123968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260434432, + "active_bytes_peak_delta": 3801088, + "after": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113260434432, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113260434432, + "device": 4, + "inactive_split_bytes_current": 8275346432, + "inactive_split_bytes_peak": 11918432256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:44.301420Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855123968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260434432, + "allocated_bytes_peak_delta": 3801088, + "before": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113256633344, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113256633344, + "device": 4, + "inactive_split_bytes_current": 8264860672, + "inactive_split_bytes_peak": 11918432256, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:43.259159Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8275346432, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 11918432256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117190950912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117190950912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 524, + "segment_count_current_delta": 0, + "segment_count_peak": 524, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T142544301819Z.pickle", + "timestamp_utc": "2026-08-01T14:25:44.396901Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T142544301819Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T142544301819Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ab1c1a50a2989f0063987acb709a6407f8d3c52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T142544301819Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d74caeb1c8742c997ca1c2fd4928864e14935de0c38b545d9178c9bd6bfc5e5f +size 5614280 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T142545460264Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T142545460264Z.json new file mode 100644 index 0000000000000000000000000000000000000000..09bbff221b534f85e8649ba7551cac253ba3b709 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T142545460264Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855123968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260434432, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113260434432, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113260434432, + "device": 4, + "inactive_split_bytes_current": 8264860672, + "inactive_split_bytes_peak": 11918432256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:45.459845Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855123968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260434432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113260434432, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113260434432, + "device": 4, + "inactive_split_bytes_current": 8275346432, + "inactive_split_bytes_peak": 11918432256, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:44.408065Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8264860672, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 11918432256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117190950912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117190950912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 524, + "segment_count_current_delta": 0, + "segment_count_peak": 524, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T142545460264Z.pickle", + "timestamp_utc": "2026-08-01T14:25:45.557861Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T142545460264Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T142545460264Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fc4e0b86c091fa3888f9555f6787190cd256c8d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T142545460264Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d29563562c8df06a9333302eff70f6dc89f6c9cd5a5bdaab57c05254928ce8db +size 5650647 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T142546616680Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T142546616680Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a02a09327f6c196336bd91212b485fbb36f38d8c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T142546616680Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855123968, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113260434432, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113260434432, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113260434432, + "device": 4, + "inactive_split_bytes_current": 8503936000, + "inactive_split_bytes_peak": 11918432256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:46.616274Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855123968, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113260434432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113260434432, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113260434432, + "device": 4, + "inactive_split_bytes_current": 8264860672, + "inactive_split_bytes_peak": 11918432256, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:45.569334Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8503936000, + "inactive_split_bytes_current_delta": 239075328, + "inactive_split_bytes_peak": 11918432256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117190950912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117190950912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 524, + "segment_count_current_delta": 0, + "segment_count_peak": 524, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T142546616680Z.pickle", + "timestamp_utc": "2026-08-01T14:25:46.710254Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T142546616680Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T142546616680Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5af81645aba846c3abb599fd5d0929f0a75cac28 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T142546616680Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b09bd607f25a50818fb309fba58c0d14787b071a237c2b8fc41ea887e5da8dc1 +size 5617162 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T142639862910Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T142639862910Z.json new file mode 100644 index 0000000000000000000000000000000000000000..979ec1ad88fdfc4bc58fc1f2585864114dbc3171 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T142639862910Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804481024, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113260434432, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66804481024, + "active_bytes_peak": 113260434432, + "allocated_bytes_current": 66804481024, + "allocated_bytes_peak": 113260434432, + "device": 4, + "inactive_split_bytes_current": 7996736512, + "inactive_split_bytes_peak": 11918432256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:26:39.862451Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804481024, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113260434432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855123968, + "active_bytes_peak": 113260434432, + "allocated_bytes_current": 66855123968, + "allocated_bytes_peak": 113260434432, + "device": 4, + "inactive_split_bytes_current": 8503936000, + "inactive_split_bytes_peak": 11918432256, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117190950912, + "reserved_bytes_peak": 117190950912, + "segment_count_current": 524, + "segment_count_peak": 524, + "timestamp_utc": "2026-08-01T14:25:46.721261Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7996736512, + "inactive_split_bytes_current_delta": -507199488, + "inactive_split_bytes_peak": 11918432256, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117190950912, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117190950912, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 524, + "segment_count_current_delta": 0, + "segment_count_peak": 524, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T142639862910Z.pickle", + "timestamp_utc": "2026-08-01T14:26:40.067387Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T142639862910Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T142639862910Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..07611d8e91dd7daad2cf0ae8026611bcb5d96ce8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T142639862910Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:466900bf6017fd968af03ee9d4ee22c01f3f799bee71a9a8fd2e4e37eea7460e +size 9434762 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T142536998142Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T142536998142Z.json new file mode 100644 index 0000000000000000000000000000000000000000..afa05b4ed3874ea82885ce05565eb221d21f5325 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T142536998142Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 89621532672, + "active_bytes_peak_delta": 54155196416, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89621532672, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89621532672, + "device": 5, + "inactive_split_bytes_current": 8599882752, + "inactive_split_bytes_peak": 10951652352, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 98314485760, + "reserved_bytes_peak": 98314485760, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:25:36.997553Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 89621532672, + "allocated_bytes_peak_delta": 54155196416, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:24:29.932367Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8599882752, + "inactive_split_bytes_current_delta": 8467063808, + "inactive_split_bytes_peak": 10951652352, + "inactive_split_bytes_peak_delta": 10791266304, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 98314485760, + "reserved_bytes_current_delta": 59070480384, + "reserved_bytes_peak": 98314485760, + "reserved_bytes_peak_delta": 59070480384, + "segment_count_current": 463, + "segment_count_current_delta": 392, + "segment_count_peak": 463, + "segment_count_peak_delta": 392, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T142536998142Z.pickle", + "timestamp_utc": "2026-08-01T14:25:37.109749Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T142536998142Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T142536998142Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6ed9ec6a173a54ebf327a842726889bd2148611 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T142536998142Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7023e801cffb9960e52ba93e97aac58f42daef4029b15aead20397c432003e92 +size 5395411 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T142538188632Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T142538188632Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15719fc5c6dfa7f63754a33d2b8f71614599fcd4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T142538188632Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975680, + "active_bytes_peak_delta": 23638443008, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975680, + "device": 5, + "inactive_split_bytes_current": 8381778944, + "inactive_split_bytes_peak": 11943503872, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:38.188225Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975680, + "allocated_bytes_peak_delta": 23638443008, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89621532672, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89621532672, + "device": 5, + "inactive_split_bytes_current": 8599882752, + "inactive_split_bytes_peak": 10951652352, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 98314485760, + "reserved_bytes_peak": 98314485760, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:25:37.129182Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8381778944, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 11943503872, + "inactive_split_bytes_peak_delta": 991851520, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117159493632, + "reserved_bytes_current_delta": 18845007872, + "reserved_bytes_peak": 117159493632, + "reserved_bytes_peak_delta": 18845007872, + "segment_count_current": 522, + "segment_count_current_delta": 59, + "segment_count_peak": 522, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T142538188632Z.pickle", + "timestamp_utc": "2026-08-01T14:25:38.300912Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T142538188632Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T142538188632Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9da826bb1aec55a88c973a38493ddaec01e0f2ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T142538188632Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:070339da5667fbdb50559c64524363bf063b35a16533a9049d352a3330a2455b +size 5589799 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T142539355261Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T142539355261Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7934ffa33f301a63dc6da1d98a2f06a08ef36a66 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T142539355261Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975680, + "device": 5, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 11943503872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:39.354863Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975680, + "device": 5, + "inactive_split_bytes_current": 8381778944, + "inactive_split_bytes_peak": 11943503872, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:38.316095Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11943503872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117159493632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117159493632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T142539355261Z.pickle", + "timestamp_utc": "2026-08-01T14:25:39.460606Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T142539355261Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T142539355261Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..16dfbc6d32a9199c2f9b0aa5e24844ea6648bfd7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T142539355261Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f09e10f24b66d773f250ea6119e11cc0e0f9c14a302be02999c933ba42e1fab +size 5588993 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T142540525980Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T142540525980Z.json new file mode 100644 index 0000000000000000000000000000000000000000..13fecb8794b89de075bd5c5e767a1d78f8e940eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T142540525980Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975680, + "device": 5, + "inactive_split_bytes_current": 8371293184, + "inactive_split_bytes_peak": 11943503872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:40.525564Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975680, + "device": 5, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 11943503872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:39.475431Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8371293184, + "inactive_split_bytes_current_delta": -8388608, + "inactive_split_bytes_peak": 11943503872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117159493632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117159493632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T142540525980Z.pickle", + "timestamp_utc": "2026-08-01T14:25:40.630427Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T142540525980Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T142540525980Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ff22eb90e4bd2fdf0001f74d97707dc7c903fb7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T142540525980Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76f6e839c7ceb608832b445fd765f95ff0cb8cb38aa7fdec70ad9649edb8c68f +size 5597406 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T142541969894Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T142541969894Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6b5fc6057de571b9ac32b258f07ed486eadf1599 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T142541969894Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259976704, + "active_bytes_peak_delta": 1024, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259976704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259976704, + "device": 5, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 11943503872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:41.969484Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259976704, + "allocated_bytes_peak_delta": 1024, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259975680, + "device": 5, + "inactive_split_bytes_current": 8371293184, + "inactive_split_bytes_peak": 11943503872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:40.645306Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11943503872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117159493632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117159493632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T142541969894Z.pickle", + "timestamp_utc": "2026-08-01T14:25:42.076496Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T142541969894Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T142541969894Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..54957177268a24cf5ea7887ff2c06fac6a72d9f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T142541969894Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4848eb0ea8c3c708eb0c8e4f9fa911c3f6a5ca3ddffc77bfca78b1d49bcd0a7c +size 5591817 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T142543143388Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T142543143388Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d7bf0010f00a74cc12500905b2ddf3fe59a24cc0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T142543143388Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259976704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259976704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259976704, + "device": 5, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 11943503872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:43.142961Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259976704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259976704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259976704, + "device": 5, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 11943503872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:42.091377Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 11943503872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117159493632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117159493632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T142543143388Z.pickle", + "timestamp_utc": "2026-08-01T14:25:43.244251Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T142543143388Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T142543143388Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d04f49d77324350f0baa8377ecaa21046594a192 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T142543143388Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:154402db3ba270f564e1eb7bcf766247df6db50ef9f5ed8058ed0ab8e7624689 +size 5618061 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T142544301793Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T142544301793Z.json new file mode 100644 index 0000000000000000000000000000000000000000..33eb041dd6fe5ef056c10c82a64278c6de90d4f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T142544301793Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259976704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259976704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259976704, + "device": 5, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 11943503872, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:44.301396Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259976704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259976704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259976704, + "device": 5, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 11943503872, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:43.258830Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 11943503872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117159493632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117159493632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T142544301793Z.pickle", + "timestamp_utc": "2026-08-01T14:25:44.401541Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T142544301793Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T142544301793Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..847285acf710104550b0ffd106e4b50905a3c7c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T142544301793Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f48a475c5aa70953d699dad3e0e095dc4a105984ee2be17c37023a032ed3548c +size 5588836 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T142545460216Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T142545460216Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27b39d167b14864ad2ea49546c4f4e8b85b7d9d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T142545460216Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259976704, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259976704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259976704, + "device": 5, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 11943503872, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:45.459803Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259976704, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259976704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259976704, + "device": 5, + "inactive_split_bytes_current": 8379681792, + "inactive_split_bytes_peak": 11943503872, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:44.415897Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 11943503872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117159493632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117159493632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T142545460216Z.pickle", + "timestamp_utc": "2026-08-01T14:25:45.559840Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T142545460216Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T142545460216Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8948097cd161c49ebcf4a666d10cfd6f596ce9e1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T142545460216Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f00fa27f1f01570da3ddd1fe4cdb4a1095c694d658b72802a45af705eb124122 +size 5621155 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T142546616635Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T142546616635Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f78cfd0141f2a1e8177bc14599d9519d59ce1f5c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T142546616635Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113261417472, + "active_bytes_peak_delta": 1440768, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261417472, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261417472, + "device": 5, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 11943503872, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:46.616239Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113261417472, + "allocated_bytes_peak_delta": 1440768, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113259976704, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113259976704, + "device": 5, + "inactive_split_bytes_current": 8369196032, + "inactive_split_bytes_peak": 11943503872, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:45.574261Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 11943503872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117159493632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117159493632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T142546616635Z.pickle", + "timestamp_utc": "2026-08-01T14:25:46.723229Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T142546616635Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T142546616635Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a20db2abeb1a209ebab9ab792283192e8c8a560 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T142546616635Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e619c2165d383fa07983aaf638e047301096831f672c37bb4b261b88cf2aea8 +size 5591716 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T142639862922Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T142639862922Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dffc149d0a3a2941c09b2248a8386f8a037bfd34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T142639862922Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113261417472, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 113261417472, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 113261417472, + "device": 5, + "inactive_split_bytes_current": 8103169024, + "inactive_split_bytes_peak": 11943503872, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:26:39.862436Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113261417472, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113261417472, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113261417472, + "device": 5, + "inactive_split_bytes_current": 8373390336, + "inactive_split_bytes_peak": 11943503872, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117159493632, + "reserved_bytes_peak": 117159493632, + "segment_count_current": 522, + "segment_count_peak": 522, + "timestamp_utc": "2026-08-01T14:25:46.744660Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8103169024, + "inactive_split_bytes_current_delta": -270221312, + "inactive_split_bytes_peak": 11943503872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117159493632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117159493632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 522, + "segment_count_current_delta": 0, + "segment_count_peak": 522, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T142639862922Z.pickle", + "timestamp_utc": "2026-08-01T14:26:40.064132Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T142639862922Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T142639862922Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aab296bd1ea4f0e02a85217278131df68390fe72 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T142639862922Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e43caab7cf654d7a11133024130df3249ae27c5821e8877cff9d2fdcfbd9dfad +size 9394915 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T142536998201Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T142536998201Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1383243339a817000969f9e46de99f952c7cd746 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T142536998201Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 89621663744, + "active_bytes_peak_delta": 54155327488, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89621663744, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89621663744, + "device": 6, + "inactive_split_bytes_current": 8717323264, + "inactive_split_bytes_peak": 10501379072, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98431926272, + "reserved_bytes_peak": 98431926272, + "segment_count_current": 464, + "segment_count_peak": 464, + "timestamp_utc": "2026-08-01T14:25:36.997595Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 89621663744, + "allocated_bytes_peak_delta": 54155327488, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:24:29.931530Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8717323264, + "inactive_split_bytes_current_delta": 8584504320, + "inactive_split_bytes_peak": 10501379072, + "inactive_split_bytes_peak_delta": 10340993024, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98431926272, + "reserved_bytes_current_delta": 59187920896, + "reserved_bytes_peak": 98431926272, + "reserved_bytes_peak_delta": 59187920896, + "segment_count_current": 464, + "segment_count_current_delta": 393, + "segment_count_peak": 464, + "segment_count_peak_delta": 393, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T142536998201Z.pickle", + "timestamp_utc": "2026-08-01T14:25:37.109707Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T142536998201Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T142536998201Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0758d35b43dc0ec1bd0c1f76dbb2071da100df2c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T142536998201Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb398f6bf359e505b5dde24b6d37894e95c850041b1ef0e98ed00b4849bf2707 +size 5395594 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T142538188640Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T142538188640Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cfa98071f3dbee57a172e2b9dd2178513e8de1bf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T142538188640Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113254469632, + "active_bytes_peak_delta": 23632805888, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113254469632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113254469632, + "device": 6, + "inactive_split_bytes_current": 8264338432, + "inactive_split_bytes_peak": 10998264832, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:38.188229Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113254469632, + "allocated_bytes_peak_delta": 23632805888, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 89621663744, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 89621663744, + "device": 6, + "inactive_split_bytes_current": 8717323264, + "inactive_split_bytes_peak": 10501379072, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98431926272, + "reserved_bytes_peak": 98431926272, + "segment_count_current": 464, + "segment_count_peak": 464, + "timestamp_utc": "2026-08-01T14:25:37.129532Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8264338432, + "inactive_split_bytes_current_delta": -452984832, + "inactive_split_bytes_peak": 10998264832, + "inactive_split_bytes_peak_delta": 496885760, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117482455040, + "reserved_bytes_current_delta": 19050528768, + "reserved_bytes_peak": 117482455040, + "reserved_bytes_peak_delta": 19050528768, + "segment_count_current": 523, + "segment_count_current_delta": 59, + "segment_count_peak": 523, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T142538188640Z.pickle", + "timestamp_utc": "2026-08-01T14:25:38.299106Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T142538188640Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T142538188640Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6091e6067bcfb3e0b768e192871af2ec19d67b90 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T142538188640Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2acf7b327f8c619e325e904fe918a14c568f6ff7c016b723c3a1cffe8b272d91 +size 5579825 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T142539355264Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T142539355264Z.json new file mode 100644 index 0000000000000000000000000000000000000000..29391ed7b971fc9d7eeacb5edf89a965fd367baf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T142539355264Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113265873408, + "active_bytes_peak_delta": 11403776, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8262241280, + "inactive_split_bytes_peak": 10998264832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:39.354865Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113265873408, + "allocated_bytes_peak_delta": 11403776, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113254469632, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113254469632, + "device": 6, + "inactive_split_bytes_current": 8264338432, + "inactive_split_bytes_peak": 10998264832, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:38.314390Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8262241280, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 10998264832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117482455040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117482455040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T142539355264Z.pickle", + "timestamp_utc": "2026-08-01T14:25:39.458566Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T142539355264Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T142539355264Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f1f8add0fe3fdb0440bd0d1414df893e909a9c23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T142539355264Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86080cf567409de307693319c72d7bcdb79ebcb33fed6b1269ffc07a91611b6a +size 5589516 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T142540525939Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T142540525939Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7950613962ba15f290088b0c7309207e8ba224b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T142540525939Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113265873408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8253852672, + "inactive_split_bytes_peak": 10998264832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:40.525539Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113265873408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8262241280, + "inactive_split_bytes_peak": 10998264832, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:39.473676Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8253852672, + "inactive_split_bytes_current_delta": -8388608, + "inactive_split_bytes_peak": 10998264832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117482455040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117482455040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T142540525939Z.pickle", + "timestamp_utc": "2026-08-01T14:25:40.629776Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T142540525939Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T142540525939Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93090bc20e2cd9d171c6a1cf31fed123d8d02b92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T142540525939Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eaf3e3302b46ad426df966531c6c5fff23ef38fc30e1e0521583689c86ac83b5 +size 5606896 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T142541969894Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T142541969894Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b845cd5beadbe8b01ae5fbc6d00af2305b1f12ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T142541969894Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113265873408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8255949824, + "inactive_split_bytes_peak": 10998264832, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:41.969489Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113265873408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8253852672, + "inactive_split_bytes_peak": 10998264832, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:40.644684Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8255949824, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 10998264832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117482455040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117482455040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T142541969894Z.pickle", + "timestamp_utc": "2026-08-01T14:25:42.077442Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T142541969894Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T142541969894Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e455a394c6f21988b1a7262b77419650d8eb46a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T142541969894Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed4f9d6ad153a40082ae7f016cb4e28b1941c9d06dc0e59c59b313cd0b765d59 +size 5592230 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T142543143329Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T142543143329Z.json new file mode 100644 index 0000000000000000000000000000000000000000..200394ab851dd33a08cf9b831d8fb75fb653fb8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T142543143329Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113265873408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8251755520, + "inactive_split_bytes_peak": 10998264832, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:43.142929Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113265873408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8255949824, + "inactive_split_bytes_peak": 10998264832, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:42.092203Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8251755520, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 10998264832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117482455040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117482455040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T142543143329Z.pickle", + "timestamp_utc": "2026-08-01T14:25:43.244061Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T142543143329Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T142543143329Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..30b0b6b742e4c3338e6d6b96c582f2580cf93877 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T142543143329Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce4461cbade215b21fae137c1851b41b316b6c7a02e483e12d10b2565a8d69b9 +size 5621742 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T142544301790Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T142544301790Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4797609f74dddbd9cd653ba2dd018220579dc22d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T142544301790Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113265873408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8262241280, + "inactive_split_bytes_peak": 10998264832, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:44.301387Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113265873408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8251755520, + "inactive_split_bytes_peak": 10998264832, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:43.258704Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8262241280, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 10998264832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117482455040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117482455040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T142544301790Z.pickle", + "timestamp_utc": "2026-08-01T14:25:44.401733Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T142544301790Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T142544301790Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..77987aa2de2a079111ada1200e4d92b77074fd31 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T142544301790Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c03460fd3876f3f1ad46a60ae39312095ed91ab10c37a8c91dafb09d4714a7d +size 5589328 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T142545460216Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T142545460216Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9ac44e201b929bdb55c8438752dd64b817ab393f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T142545460216Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113265873408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8251755520, + "inactive_split_bytes_peak": 10998264832, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:45.459812Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113265873408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8262241280, + "inactive_split_bytes_peak": 10998264832, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:44.416432Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8251755520, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 10998264832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117482455040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117482455040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T142545460216Z.pickle", + "timestamp_utc": "2026-08-01T14:25:45.561999Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T142545460216Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T142545460216Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b5e96b5c95f2b2e07cdeb986875f203aeb68803e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T142545460216Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f26562475af7cc406effb143502ed9570e6bd39098906a2a7817d8bfbef216e +size 5621610 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T142546616624Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T142546616624Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c35545aad6971ee9a39d80405cbaee67129fec10 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T142546616624Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113265873408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8255949824, + "inactive_split_bytes_peak": 10998264832, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:46.616225Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113265873408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8251755520, + "inactive_split_bytes_peak": 10998264832, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:45.576844Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8255949824, + "inactive_split_bytes_current_delta": 4194304, + "inactive_split_bytes_peak": 10998264832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117482455040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117482455040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T142546616624Z.pickle", + "timestamp_utc": "2026-08-01T14:25:46.720414Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T142546616624Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T142546616624Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14cd20aaa657593cd1e92d2676ff8277ed15036a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T142546616624Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe729ae16cbb8339925c2d01cc3428ba283f3b1d59f615b18c6f0627d2ee0cd4 +size 5592137 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T142639862896Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T142639862896Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0fa0be9ad7bb4dd7c1bf142b0707bc65e53f6c3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T142639862896Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113265873408, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 7983631360, + "inactive_split_bytes_peak": 10998264832, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:26:39.862433Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113265873408, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 113265873408, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 113265873408, + "device": 6, + "inactive_split_bytes_current": 8255949824, + "inactive_split_bytes_peak": 10998264832, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117482455040, + "reserved_bytes_peak": 117482455040, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:46.737391Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7983631360, + "inactive_split_bytes_current_delta": -272318464, + "inactive_split_bytes_peak": 10998264832, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117482455040, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117482455040, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T142639862896Z.pickle", + "timestamp_utc": "2026-08-01T14:26:40.058221Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T142639862896Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T142639862896Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b83f5e521f2f88af5acc52a719008e74969b86a4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T142639862896Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa8bbdca20611a8a0d24aab3a2bce05755fc9183112d904822a5e4d1fc05a2c4 +size 9395335 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T142536998103Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T142536998103Z.json new file mode 100644 index 0000000000000000000000000000000000000000..11873438ad9ac3eb282b8ee8586d6741e87132ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T142536998103Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 31622094336, + "active_bytes_peak": 89621532672, + "active_bytes_peak_delta": 54155196416, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 89621532672, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 89621532672, + "device": 7, + "inactive_split_bytes_current": 8599882240, + "inactive_split_bytes_peak": 10437268992, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 98314485760, + "reserved_bytes_peak": 98314485760, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:25:36.997617Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 31622094336, + "allocated_bytes_peak": 89621532672, + "allocated_bytes_peak_delta": 54155196416, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:24:29.931324Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8599882240, + "inactive_split_bytes_current_delta": 8467063296, + "inactive_split_bytes_peak": 10437268992, + "inactive_split_bytes_peak_delta": 10276882944, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 98314485760, + "reserved_bytes_current_delta": 59070480384, + "reserved_bytes_peak": 98314485760, + "reserved_bytes_peak_delta": 59070480384, + "segment_count_current": 463, + "segment_count_current_delta": 392, + "segment_count_peak": 463, + "segment_count_peak_delta": 392, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T142536998103Z.pickle", + "timestamp_utc": "2026-08-01T14:25:37.109527Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T142536998103Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T142536998103Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0b8f9d2fb2a374020a20a0d7843aff0d72f3ff79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T142536998103Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e4489e14891c9c3dcde52b44f474b84d360c3eff8453cdb91a839991be92466 +size 5397006 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T142538188691Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T142538188691Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a0f333a09d957848a64ce2e6983d6734da0c8106 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T142538188691Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259713024, + "active_bytes_peak_delta": 23638180352, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259713024, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259713024, + "device": 7, + "inactive_split_bytes_current": 8381778432, + "inactive_split_bytes_peak": 10907641344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:38.188292Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259713024, + "allocated_bytes_peak_delta": 23638180352, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 89621532672, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 89621532672, + "device": 7, + "inactive_split_bytes_current": 8599882240, + "inactive_split_bytes_peak": 10437268992, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 98314485760, + "reserved_bytes_peak": 98314485760, + "segment_count_current": 463, + "segment_count_peak": 463, + "timestamp_utc": "2026-08-01T14:25:37.127021Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8381778432, + "inactive_split_bytes_current_delta": -218103808, + "inactive_split_bytes_peak": 10907641344, + "inactive_split_bytes_peak_delta": 470372352, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 18903728128, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 18903728128, + "segment_count_current": 523, + "segment_count_current_delta": 60, + "segment_count_peak": 523, + "segment_count_peak_delta": 60, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T142538188691Z.pickle", + "timestamp_utc": "2026-08-01T14:25:38.298942Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T142538188691Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T142538188691Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d861a417b575a876b4ee8301fe8c1a608f115d00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T142538188691Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3a512fb097821cff22915eac5d416904d197c7a35457b553d94d13a9071d9c3 +size 5582082 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T142539355312Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T142539355312Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ddd001d0679cca63d1daff5712bfbe781450e2c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T142539355312Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975680, + "active_bytes_peak_delta": 262656, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8379681280, + "inactive_split_bytes_peak": 11898850816, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:39.354907Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975680, + "allocated_bytes_peak_delta": 262656, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259713024, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259713024, + "device": 7, + "inactive_split_bytes_current": 8381778432, + "inactive_split_bytes_peak": 10907641344, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:38.312154Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8379681280, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11898850816, + "inactive_split_bytes_peak_delta": 991209472, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T142539355312Z.pickle", + "timestamp_utc": "2026-08-01T14:25:39.456818Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T142539355312Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T142539355312Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..508c19ebb2acf7e8e963c891c46ca3bd59f4ee1f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T142539355312Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89d62c49e00ae372888886fcafe2728723df743ce969a6a628e57c130122aa89 +size 5591181 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T142540525981Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T142540525981Z.json new file mode 100644 index 0000000000000000000000000000000000000000..966b2eaa56978feec1b014467e874c9f33bd3b9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T142540525981Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8371292672, + "inactive_split_bytes_peak": 11898850816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:40.525577Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8379681280, + "inactive_split_bytes_peak": 11898850816, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:39.469671Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8371292672, + "inactive_split_bytes_current_delta": -8388608, + "inactive_split_bytes_peak": 11898850816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T142540525981Z.pickle", + "timestamp_utc": "2026-08-01T14:25:40.627696Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T142540525981Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T142540525981Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4bd445c26abfa261da3064f9f3a38b1f7dae1215 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T142540525981Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bac867b3b957c1b4619cde4d1f1e1f0f12d62a5f0f119824387e7bfb93aaaef +size 5599519 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T142541969922Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T142541969922Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f07bace15ffb7a1ccbde946b47d885987d890f13 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T142541969922Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8373389824, + "inactive_split_bytes_peak": 11898850816, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:41.969502Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8371292672, + "inactive_split_bytes_peak": 11898850816, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:40.640431Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8373389824, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11898850816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T142541969922Z.pickle", + "timestamp_utc": "2026-08-01T14:25:42.075709Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T142541969922Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T142541969922Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5ffd37a38c785196a36a798803cdbdef113983cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T142541969922Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cca1194d8ecfcb990b5b34a94cfd43594257064c7fe7bb49fbc7d2a9033a661 +size 5584139 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T142543143329Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T142543143329Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c52261d3541f12bc7953b8284417720c6ca5035e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T142543143329Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8369195520, + "inactive_split_bytes_peak": 11898850816, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:43.142935Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8373389824, + "inactive_split_bytes_peak": 11898850816, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:42.088524Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8369195520, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 11898850816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T142543143329Z.pickle", + "timestamp_utc": "2026-08-01T14:25:43.245440Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T142543143329Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T142543143329Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ffc6594527f15d1dee950ce8dd0b0e62bee20beb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T142543143329Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e76f95a120f3c0c3fae7c24018657b0110516416f4a383ca8140db22aec1a242 +size 5620262 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T142544301839Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T142544301839Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9a846eca6f25a0e5ef8f6d3dafb7fe3fd6387f64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T142544301839Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8379681280, + "inactive_split_bytes_peak": 11898850816, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:44.301439Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8369195520, + "inactive_split_bytes_peak": 11898850816, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:43.260281Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8379681280, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 11898850816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T142544301839Z.pickle", + "timestamp_utc": "2026-08-01T14:25:44.397750Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T142544301839Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T142544301839Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..85eeb05b5b4191bd9ff37945d2b6eaf37217e626 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T142544301839Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f9d92643a67a1c5ad9970d1eda6294e895b8ab992ced2e9708734b5e78d34da +size 5590995 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T142545460260Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T142545460260Z.json new file mode 100644 index 0000000000000000000000000000000000000000..34b83c5c6756178649648fbd6059850da03f3351 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T142545460260Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8369195520, + "inactive_split_bytes_peak": 11898850816, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:45.459859Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8379681280, + "inactive_split_bytes_peak": 11898850816, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:44.409959Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8369195520, + "inactive_split_bytes_current_delta": -10485760, + "inactive_split_bytes_peak": 11898850816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T142545460260Z.pickle", + "timestamp_utc": "2026-08-01T14:25:45.557294Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T142545460260Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T142545460260Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8cd6a20aa011292b1c1181a3a06a6ae841b6dd3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T142545460260Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f29ec36f742c59a1cc8006fb5b2015774ef3c4766907afed65b0ac78ba74a22 +size 5623272 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T142546616679Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T142546616679Z.json new file mode 100644 index 0000000000000000000000000000000000000000..815dddc1f24ca10e109cfe25ea3bbe5ab33ee8ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T142546616679Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113259975680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8610368000, + "inactive_split_bytes_peak": 12024417792, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:46.616278Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113259975680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8369195520, + "inactive_split_bytes_peak": 11898850816, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:45.569614Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8610368000, + "inactive_split_bytes_current_delta": 241172480, + "inactive_split_bytes_peak": 12024417792, + "inactive_split_bytes_peak_delta": 125566976, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T142546616679Z.pickle", + "timestamp_utc": "2026-08-01T14:25:46.712614Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T142546616679Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T142546616679Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7f42fd294f70edce69755ea9fc82ff389453ebbf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T142546616679Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:441e35e7011173565be29dcdc6e34060fb6bafd8aee2e1f7ec08ee34787d54ca +size 5584096 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T142639862889Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T142639862889Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e21bd1f24173e7e3bd38d1ab96dfea37b4fa468f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T142639862889Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906624, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 113259975680, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906624, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66802906624, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8103168512, + "inactive_split_bytes_peak": 12024417792, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:26:39.862431Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906624, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 113259975680, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 113259975680, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 113259975680, + "device": 7, + "inactive_split_bytes_current": 8610368000, + "inactive_split_bytes_peak": 12024417792, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117218213888, + "reserved_bytes_peak": 117218213888, + "segment_count_current": 523, + "segment_count_peak": 523, + "timestamp_utc": "2026-08-01T14:25:46.724904Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8103168512, + "inactive_split_bytes_current_delta": -507199488, + "inactive_split_bytes_peak": 12024417792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117218213888, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117218213888, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 523, + "segment_count_current_delta": 0, + "segment_count_peak": 523, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T142639862889Z.pickle", + "timestamp_utc": "2026-08-01T14:26:40.058797Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T142639862889Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T142639862889Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..31539d73a85b0cb91c8bed1c3bd83d126e20fdcf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T142639862889Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7fe6f2b6ed213e1a272c49da67eb2373e80b922f91b96e3ea4aa80033e4728f +size 9383491 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..17ea18424261b005d30ec5a73b6a4f78048f28fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 4096 +capacity: 1.25 +expert_dispatch_overlap: on +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6043 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 230 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T142923990389Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T142923990389Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6628d23a70bd55e3bd4db35b7ef4ed91eda79900 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T142923990389Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441598976, + "active_bytes_current_delta": 37210143744, + "active_bytes_peak": 89878631936, + "active_bytes_peak_delta": 54412295680, + "after": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 89878631936, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 89526310400, + "device": 0, + "inactive_split_bytes_current": 8414096384, + "inactive_split_bytes_peak": 13653190656, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 99704897536, + "reserved_bytes_peak": 99704897536, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T14:29:23.989811Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441598976, + "allocated_bytes_current_delta": 37210143744, + "allocated_bytes_peak": 89526310400, + "allocated_bytes_peak_delta": 54059974144, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T14:28:15.713399Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8414096384, + "inactive_split_bytes_current_delta": 8279180288, + "inactive_split_bytes_peak": 13653190656, + "inactive_split_bytes_peak_delta": 13492558848, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 99704897536, + "reserved_bytes_current_delta": 60458795008, + "reserved_bytes_peak": 99704897536, + "reserved_bytes_peak_delta": 60458795008, + "segment_count_current": 420, + "segment_count_current_delta": 348, + "segment_count_peak": 420, + "segment_count_peak_delta": 348, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T142923990389Z.pickle", + "timestamp_utc": "2026-08-01T14:29:24.112115Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T142923990389Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T142923990389Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cdd767bed80245c48c89d8367e196e210591f769 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T142923990389Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ad89f4fe827d077834409a418b71e07020185da57695bc29d7b0a1403ebdffb +size 7088797 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T142925433552Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T142925433552Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06388d311e2175dbddb82b7d8057188ea4516b8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T142925433552Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441598976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113509143040, + "active_bytes_peak_delta": 23630511104, + "after": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113509143040, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113156821504, + "device": 0, + "inactive_split_bytes_current": 11352206336, + "inactive_split_bytes_peak": 14235962880, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 120424759296, + "reserved_bytes_peak": 120424759296, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T14:29:25.433138Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441598976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156821504, + "allocated_bytes_peak_delta": 23630511104, + "before": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 89878631936, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 89526310400, + "device": 0, + "inactive_split_bytes_current": 8414096384, + "inactive_split_bytes_peak": 13653190656, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 99704897536, + "reserved_bytes_peak": 99704897536, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T14:29:24.132283Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11352206336, + "inactive_split_bytes_current_delta": 2938109952, + "inactive_split_bytes_peak": 14235962880, + "inactive_split_bytes_peak_delta": 582772224, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 120424759296, + "reserved_bytes_current_delta": 20719861760, + "reserved_bytes_peak": 120424759296, + "reserved_bytes_peak_delta": 20719861760, + "segment_count_current": 487, + "segment_count_current_delta": 67, + "segment_count_peak": 487, + "segment_count_peak_delta": 67, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T142925433552Z.pickle", + "timestamp_utc": "2026-08-01T14:29:25.546510Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T142925433552Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T142925433552Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..703fe1642c95b5c2d3ce982b46e173872eb490d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T142925433552Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c41d284646ecd67dbdf669f8b23c88a6ec1f4d866b32e5c2973a10cf1f6707d1 +size 5981633 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T142926766750Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T142926766750Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5aef54831ec13949316e87a787fddeceffd58387 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T142926766750Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441598976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113861886976, + "active_bytes_peak_delta": 352743936, + "after": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113861886976, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113157243904, + "device": 0, + "inactive_split_bytes_current": 10941164544, + "inactive_split_bytes_peak": 14235962880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 120424759296, + "reserved_bytes_peak": 120424759296, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T14:29:26.766350Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441598976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157243904, + "allocated_bytes_peak_delta": 422400, + "before": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113509143040, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113156821504, + "device": 0, + "inactive_split_bytes_current": 11352206336, + "inactive_split_bytes_peak": 14235962880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 120424759296, + "reserved_bytes_peak": 120424759296, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T14:29:25.560914Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10941164544, + "inactive_split_bytes_current_delta": -411041792, + "inactive_split_bytes_peak": 14235962880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 120424759296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120424759296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T142926766750Z.pickle", + "timestamp_utc": "2026-08-01T14:29:26.872094Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T142926766750Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T142926766750Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..02c3cc4cda94611e81fda3250f86da3fb61920de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T142926766750Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ade0a375ad08fb759af9314c1c91a147c8b46a314cf38b60d61a9dde37e8cc0d +size 5978713 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T142928139902Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T142928139902Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f8bbe045a57b347058113de04af153722719a1a0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T142928139902Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441598976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113864634880, + "active_bytes_peak_delta": 2747904, + "after": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113864634880, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113159991808, + "device": 0, + "inactive_split_bytes_current": 10882444288, + "inactive_split_bytes_peak": 14235962880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 120424759296, + "reserved_bytes_peak": 120424759296, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T14:29:28.139495Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441598976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113159991808, + "allocated_bytes_peak_delta": 2747904, + "before": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113861886976, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113157243904, + "device": 0, + "inactive_split_bytes_current": 10941164544, + "inactive_split_bytes_peak": 14235962880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 120424759296, + "reserved_bytes_peak": 120424759296, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T14:29:26.887515Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10882444288, + "inactive_split_bytes_current_delta": -58720256, + "inactive_split_bytes_peak": 14235962880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 120424759296, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120424759296, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T142928139902Z.pickle", + "timestamp_utc": "2026-08-01T14:29:28.243845Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T142928139902Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T142928139902Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a2baec299277a0d3b13d6ae2767f6c6c5cfb6fb3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T142928139902Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8621e0db6c9e7679c8e3725359c5a6a2fd2c55181e9fa2f60c56ff9a6ca167ae +size 5996420 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T142929737157Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T142929737157Z.json new file mode 100644 index 0000000000000000000000000000000000000000..17b2876e502cce73fb933862d69b3af58334aae5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T142929737157Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441598976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113864634880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113864634880, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113159991808, + "device": 0, + "inactive_split_bytes_current": 10586745856, + "inactive_split_bytes_peak": 14502616576, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_peak": 121248940032, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:29:29.736753Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441598976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113159991808, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113864634880, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113159991808, + "device": 0, + "inactive_split_bytes_current": 10882444288, + "inactive_split_bytes_peak": 14235962880, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 120424759296, + "reserved_bytes_peak": 120424759296, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T14:29:28.257417Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10586745856, + "inactive_split_bytes_current_delta": -295698432, + "inactive_split_bytes_peak": 14502616576, + "inactive_split_bytes_peak_delta": 266653696, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 121248940032, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 488, + "segment_count_current_delta": 1, + "segment_count_peak": 488, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T142929737157Z.pickle", + "timestamp_utc": "2026-08-01T14:29:29.845051Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T142929737157Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T142929737157Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..442240d2bc885fa1e97b8fd18a97bbf704b2f35c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T142929737157Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e18553ee4aaf8ddee6bf6877c3fefc9f43821cf207788ad6b502b6d6c3c2618f +size 5977598 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T142931061260Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T142931061260Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8700f4e6826b7ce767eab77c1b467159d513ceec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T142931061260Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441598976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113864634880, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113864634880, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113159991808, + "device": 0, + "inactive_split_bytes_current": 10997787648, + "inactive_split_bytes_peak": 14502616576, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_peak": 121248940032, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:29:31.060860Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441598976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113159991808, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113864634880, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113159991808, + "device": 0, + "inactive_split_bytes_current": 10586745856, + "inactive_split_bytes_peak": 14502616576, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_peak": 121248940032, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:29:29.858862Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10997787648, + "inactive_split_bytes_current_delta": 411041792, + "inactive_split_bytes_peak": 14502616576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121248940032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T142931061260Z.pickle", + "timestamp_utc": "2026-08-01T14:29:31.165085Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T142931061260Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T142931061260Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..230642db10cf9778ebcd215e5c1eff3b73c21217 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T142931061260Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58917b1450f74fef77d2ff41011d952fbf6a2bc24e0f1e4dc52ea9d3ee552397 +size 5978258 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T142932381107Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T142932381107Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15a1e93b6e468fe7c119d762143a98bb35f5d40d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T142932381107Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441598976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113869083648, + "active_bytes_peak_delta": 4448768, + "after": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113869083648, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113164440576, + "device": 0, + "inactive_split_bytes_current": 10765003776, + "inactive_split_bytes_peak": 14502616576, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_peak": 121248940032, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:29:32.380699Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441598976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113164440576, + "allocated_bytes_peak_delta": 4448768, + "before": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113864634880, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113159991808, + "device": 0, + "inactive_split_bytes_current": 10997787648, + "inactive_split_bytes_peak": 14502616576, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_peak": 121248940032, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:29:31.178611Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10765003776, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 14502616576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121248940032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T142932381107Z.pickle", + "timestamp_utc": "2026-08-01T14:29:32.483333Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T142932381107Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T142932381107Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..12413c3728dee9754c9dc4074bb32a02e3c9e10e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T142932381107Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:788b6b4cfde5cda98cb0c0f2abe715e9ae3f6274113252f2f52fb5347bddb15e +size 5977927 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T142933937746Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T142933937746Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed1883f79ce01805c1385c39c6c37f6106422b22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T142933937746Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441598976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113869083648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113869083648, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113164440576, + "device": 0, + "inactive_split_bytes_current": 10762906624, + "inactive_split_bytes_peak": 14502616576, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_peak": 121248940032, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:29:33.937343Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441598976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113164440576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113869083648, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113164440576, + "device": 0, + "inactive_split_bytes_current": 10765003776, + "inactive_split_bytes_peak": 14502616576, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_peak": 121248940032, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:29:32.497363Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 10762906624, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 14502616576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121248940032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T142933937746Z.pickle", + "timestamp_utc": "2026-08-01T14:29:34.041202Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T142933937746Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T142933937746Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aba70a05314799c363e077fc17a34b8190562da7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T142933937746Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:606aba9feb3858532622a65a3579745a650e5a9a64a76c4a883f604576170ce7 +size 5977453 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T142935269656Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T142935269656Z.json new file mode 100644 index 0000000000000000000000000000000000000000..544cf43730bf6827a9a570c7dae8a4b1bce63f7e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T142935269656Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441598976, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113869083648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113869083648, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113164440576, + "device": 0, + "inactive_split_bytes_current": 11056507904, + "inactive_split_bytes_peak": 14601337344, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_peak": 121248940032, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:29:35.269239Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441598976, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113164440576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113869083648, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113164440576, + "device": 0, + "inactive_split_bytes_current": 10762906624, + "inactive_split_bytes_peak": 14502616576, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_peak": 121248940032, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:29:34.055251Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11056507904, + "inactive_split_bytes_current_delta": 293601280, + "inactive_split_bytes_peak": 14601337344, + "inactive_split_bytes_peak_delta": 98720768, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121248940032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T142935269656Z.pickle", + "timestamp_utc": "2026-08-01T14:29:35.372445Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T142935269656Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T142935269656Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c46b3bd1cbca961d659415d53b3da1dfa7ff4dd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T142935269656Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94ea87a44c7d028791655918b7217e035bab70b5558498eaca5c2b553f57a5f0 +size 5977534 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T143034050688Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T143034050688Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2005ddb7f3cc11cdf4b2805c59890023f202cfff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T143034050688Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61167309824, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113869083648, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61167309824, + "active_bytes_peak": 113869083648, + "allocated_bytes_current": 61167309824, + "allocated_bytes_peak": 113164440576, + "device": 0, + "inactive_split_bytes_current": 11759053824, + "inactive_split_bytes_peak": 15072558592, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_peak": 121248940032, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:30:34.050235Z", + "world_size": 8 + }, + "allocated_bytes_current": 61167309824, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113164440576, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441598976, + "active_bytes_peak": 113869083648, + "allocated_bytes_current": 72441598976, + "allocated_bytes_peak": 113164440576, + "device": 0, + "inactive_split_bytes_current": 11056507904, + "inactive_split_bytes_peak": 14601337344, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_peak": 121248940032, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:29:35.386496Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 11759053824, + "inactive_split_bytes_current_delta": 702545920, + "inactive_split_bytes_peak": 15072558592, + "inactive_split_bytes_peak_delta": 471221248, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 121248940032, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121248940032, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T143034050688Z.pickle", + "timestamp_utc": "2026-08-01T14:30:34.250943Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T143034050688Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T143034050688Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..03e566ab2a6d56ce97fbb57d9b9243ec193e63c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T143034050688Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5337c4984c429975baff00a39b80d573253b47b2ea5d8ff9fa87b4b9c8bc2e05 +size 9781475 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T142923990154Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T142923990154Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c457fb6b9b28ab37fe0d7f9f135b8c5e3fcf4ee4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T142923990154Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 89876531712, + "active_bytes_peak_delta": 54410195456, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 89876531712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 89524210176, + "device": 1, + "inactive_split_bytes_current": 8130432000, + "inactive_split_bytes_peak": 12533052416, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 99700703232, + "reserved_bytes_peak": 99700703232, + "segment_count_current": 419, + "segment_count_peak": 419, + "timestamp_utc": "2026-08-01T14:29:23.989556Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 89524210176, + "allocated_bytes_peak_delta": 54057873920, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:28:15.713341Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8130432000, + "inactive_split_bytes_current_delta": 7997613056, + "inactive_split_bytes_peak": 12533052416, + "inactive_split_bytes_peak_delta": 12372666368, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 99700703232, + "reserved_bytes_current_delta": 60456697856, + "reserved_bytes_peak": 99700703232, + "reserved_bytes_peak_delta": 60456697856, + "segment_count_current": 419, + "segment_count_current_delta": 348, + "segment_count_peak": 419, + "segment_count_peak_delta": 348, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T142923990154Z.pickle", + "timestamp_utc": "2026-08-01T14:29:24.107648Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T142923990154Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T142923990154Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bdd69223e3ceb00d7be5acc96cc17dc22335ff0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T142923990154Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f126d07e459e4b8b42d77e9988393914b3c78803fef93e13738af97e8e69c15 +size 5955080 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T142925433555Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T142925433555Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6ca0ec30ddfbe34944c7e9b769d30b06a9079fde --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T142925433555Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113508418048, + "active_bytes_peak_delta": 23631886336, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113508418048, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156096512, + "device": 1, + "inactive_split_bytes_current": 11129359360, + "inactive_split_bytes_peak": 13897447936, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120772886528, + "reserved_bytes_peak": 120772886528, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:25.433136Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156096512, + "allocated_bytes_peak_delta": 23631886336, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 89876531712, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 89524210176, + "device": 1, + "inactive_split_bytes_current": 8130432000, + "inactive_split_bytes_peak": 12533052416, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 99700703232, + "reserved_bytes_peak": 99700703232, + "segment_count_current": 419, + "segment_count_peak": 419, + "timestamp_utc": "2026-08-01T14:29:24.127735Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11129359360, + "inactive_split_bytes_current_delta": 2998927360, + "inactive_split_bytes_peak": 13897447936, + "inactive_split_bytes_peak_delta": 1364395520, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120772886528, + "reserved_bytes_current_delta": 21072183296, + "reserved_bytes_peak": 120772886528, + "reserved_bytes_peak_delta": 21072183296, + "segment_count_current": 485, + "segment_count_current_delta": 66, + "segment_count_peak": 485, + "segment_count_peak_delta": 66, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T142925433555Z.pickle", + "timestamp_utc": "2026-08-01T14:29:25.548219Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T142925433555Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T142925433555Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b155cd0febc4aef39dc6cc9bd8b5604e096a746c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T142925433555Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74ed7dd5876c717db5e0ccd14b2641930eeaa8bad92dd348116197cded429877 +size 6041881 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T142926766763Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T142926766763Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e70635080a44865a9df0730cd4d985c5ebc217f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T142926766763Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113856807936, + "active_bytes_peak_delta": 348389888, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113856807936, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156096512, + "device": 1, + "inactive_split_bytes_current": 11007724544, + "inactive_split_bytes_peak": 14362199552, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120772886528, + "reserved_bytes_peak": 120772886528, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:26.766365Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156096512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113508418048, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156096512, + "device": 1, + "inactive_split_bytes_current": 11129359360, + "inactive_split_bytes_peak": 13897447936, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 120772886528, + "reserved_bytes_peak": 120772886528, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:25.562484Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11007724544, + "inactive_split_bytes_current_delta": -121634816, + "inactive_split_bytes_peak": 14362199552, + "inactive_split_bytes_peak_delta": 464751616, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120772886528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120772886528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T142926766763Z.pickle", + "timestamp_utc": "2026-08-01T14:29:26.872404Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T142926766763Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T142926766763Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0588e7cce1a510466923d1899d935c6ae7c4ff9e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T142926766763Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9deb2d6cb42df9c885fb84269f754b4a431b7abc5cdecf70c20c762130d02844 +size 5952998 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T142928139916Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T142928139916Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7bde9b9d66e34aaf0f2856f8acc6cd1270a6d944 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T142928139916Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113858511360, + "active_bytes_peak_delta": 1703424, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113858511360, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156096512, + "device": 1, + "inactive_split_bytes_current": 11890625536, + "inactive_split_bytes_peak": 14576404992, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120772886528, + "reserved_bytes_peak": 120772886528, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:28.139527Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156096512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113856807936, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156096512, + "device": 1, + "inactive_split_bytes_current": 11007724544, + "inactive_split_bytes_peak": 14362199552, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 120772886528, + "reserved_bytes_peak": 120772886528, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:26.887611Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11890625536, + "inactive_split_bytes_current_delta": 882900992, + "inactive_split_bytes_peak": 14576404992, + "inactive_split_bytes_peak_delta": 214205440, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120772886528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120772886528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T142928139916Z.pickle", + "timestamp_utc": "2026-08-01T14:29:28.244572Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T142928139916Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T142928139916Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a5ccb2a2f15240bcb192710fe75aa310fd956d81 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T142928139916Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab8af84b9c73eee05b7f0097c81928fd9b9f149cfb8e2e5c10f81a9965aaf222 +size 5952518 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T142929737186Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T142929737186Z.json new file mode 100644 index 0000000000000000000000000000000000000000..aba8ce10968e241bd997ea69174ff95d1db3d455 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T142929737186Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113858511360, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113858511360, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156096512, + "device": 1, + "inactive_split_bytes_current": 11366337536, + "inactive_split_bytes_peak": 14576404992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120772886528, + "reserved_bytes_peak": 120772886528, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:29.736776Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156096512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113858511360, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156096512, + "device": 1, + "inactive_split_bytes_current": 11890625536, + "inactive_split_bytes_peak": 14576404992, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 120772886528, + "reserved_bytes_peak": 120772886528, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:28.258126Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11366337536, + "inactive_split_bytes_current_delta": -524288000, + "inactive_split_bytes_peak": 14576404992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120772886528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120772886528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T142929737186Z.pickle", + "timestamp_utc": "2026-08-01T14:29:29.844712Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T142929737186Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T142929737186Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d4ce1f47c2be7d5992a93ee5ffacff39bfe2cac9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T142929737186Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebb4fa44f21273059f17da6b289aec128b678bf9d3bdb71c4346eff943c287bb +size 5952067 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T142931061264Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T142931061264Z.json new file mode 100644 index 0000000000000000000000000000000000000000..213ae85404c60ba05fbfbe8c245d657d4365e8ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T142931061264Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113859056640, + "active_bytes_peak_delta": 545280, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113859056640, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156096512, + "device": 1, + "inactive_split_bytes_current": 10714123264, + "inactive_split_bytes_peak": 14576404992, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120772886528, + "reserved_bytes_peak": 120772886528, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:31.060860Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156096512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113858511360, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156096512, + "device": 1, + "inactive_split_bytes_current": 11366337536, + "inactive_split_bytes_peak": 14576404992, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 120772886528, + "reserved_bytes_peak": 120772886528, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:29.858499Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10714123264, + "inactive_split_bytes_current_delta": -652214272, + "inactive_split_bytes_peak": 14576404992, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120772886528, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120772886528, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T142931061264Z.pickle", + "timestamp_utc": "2026-08-01T14:29:31.164788Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T142931061264Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T142931061264Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8fb6e1e126831b87886357d3a3afb8e8c4d768e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T142931061264Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:592072cea9f4a996c107033828bb168584c8dff20cbe175b79cc4f93256de83b +size 5952000 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T142932381111Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T142932381111Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f53f80a6a1ac6c18f94167ff049e09f2959c1a4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T142932381111Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113859056640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113859056640, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156096512, + "device": 1, + "inactive_split_bytes_current": 12242947072, + "inactive_split_bytes_peak": 14599915008, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 121597067264, + "reserved_bytes_peak": 121597067264, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:32.380708Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156096512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113859056640, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156096512, + "device": 1, + "inactive_split_bytes_current": 10714123264, + "inactive_split_bytes_peak": 14576404992, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 120772886528, + "reserved_bytes_peak": 120772886528, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:31.178218Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12242947072, + "inactive_split_bytes_current_delta": 1528823808, + "inactive_split_bytes_peak": 14599915008, + "inactive_split_bytes_peak_delta": 23510016, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 121597067264, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 121597067264, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 486, + "segment_count_current_delta": 1, + "segment_count_peak": 486, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T142932381111Z.pickle", + "timestamp_utc": "2026-08-01T14:29:32.485037Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T142932381111Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T142932381111Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..30744b5c76b0521d22775a5f405dd5ee20438b51 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T142932381111Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38067abb1a7fd6d1fc9cf22f05fd137a1046f528b12a26dfacf45f25534d8b78 +size 6097391 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T142933937749Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T142933937749Z.json new file mode 100644 index 0000000000000000000000000000000000000000..528050978e7d42568070cc66176f63c2f8446ed2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T142933937749Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113859056640, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113859056640, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157187072, + "device": 1, + "inactive_split_bytes_current": 12125506560, + "inactive_split_bytes_peak": 14599915008, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 121597067264, + "reserved_bytes_peak": 121597067264, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:33.937350Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157187072, + "allocated_bytes_peak_delta": 1090560, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113859056640, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156096512, + "device": 1, + "inactive_split_bytes_current": 12242947072, + "inactive_split_bytes_peak": 14599915008, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 121597067264, + "reserved_bytes_peak": 121597067264, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:32.498912Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 12125506560, + "inactive_split_bytes_current_delta": -117440512, + "inactive_split_bytes_peak": 14599915008, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 121597067264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121597067264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T142933937749Z.pickle", + "timestamp_utc": "2026-08-01T14:29:34.043616Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T142933937749Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T142933937749Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..31bd2b1967d248b82e5a24437fbf0f2097ac7ae9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T142933937749Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6451ad96dbdbc4fe3b7f727e0894ce1c43b77ecf13016f269b305d63525b9729 +size 5952082 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T142935269616Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T142935269616Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2bce2197720f682b10406ff766fe1c1b548ad81c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T142935269616Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113859298304, + "active_bytes_peak_delta": 241664, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113859298304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157187072, + "device": 1, + "inactive_split_bytes_current": 11597024256, + "inactive_split_bytes_peak": 14828317184, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 121597067264, + "reserved_bytes_peak": 121597067264, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:35.269220Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157187072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113859056640, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157187072, + "device": 1, + "inactive_split_bytes_current": 12125506560, + "inactive_split_bytes_peak": 14599915008, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 121597067264, + "reserved_bytes_peak": 121597067264, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:34.057894Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11597024256, + "inactive_split_bytes_current_delta": -528482304, + "inactive_split_bytes_peak": 14828317184, + "inactive_split_bytes_peak_delta": 228402176, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 121597067264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121597067264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T142935269616Z.pickle", + "timestamp_utc": "2026-08-01T14:29:35.373916Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T142935269616Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T142935269616Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9b17b6f12933759b7beb4c60037b977df975e0c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T142935269616Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cc0eadf42f45b6a32b0575b4f36ff8e6d988badcafaeea9d2d87285dced2d22 +size 5952062 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T143034050800Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T143034050800Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac52e68c304c1617dd7a99605c8a7d58bdd140df --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T143034050800Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113859298304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 113859298304, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 113157187072, + "device": 1, + "inactive_split_bytes_current": 11768990720, + "inactive_split_bytes_peak": 14828317184, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 121597067264, + "reserved_bytes_peak": 121597067264, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:30:34.050244Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113157187072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113859298304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157187072, + "device": 1, + "inactive_split_bytes_current": 11597024256, + "inactive_split_bytes_peak": 14828317184, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 121597067264, + "reserved_bytes_peak": 121597067264, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:35.387911Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11768990720, + "inactive_split_bytes_current_delta": 171966464, + "inactive_split_bytes_peak": 14828317184, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 121597067264, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121597067264, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T143034050800Z.pickle", + "timestamp_utc": "2026-08-01T14:30:34.250971Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T143034050800Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T143034050800Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c06a8d57c496415af599421d61a7dc2e3067db67 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T143034050800Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db50d0e1158a5f8d9f36c4714c8ca350dcfb36b7bb027c5c99c8eafd3f983749 +size 9742267 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T142923990193Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T142923990193Z.json new file mode 100644 index 0000000000000000000000000000000000000000..080c345476bd7487eca9ab778c4241f256e61e4f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T142923990193Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 89874149888, + "active_bytes_peak_delta": 54407813632, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 89874149888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 89521828352, + "device": 2, + "inactive_split_bytes_current": 8394673152, + "inactive_split_bytes_peak": 11829742592, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 99994304512, + "reserved_bytes_peak": 99994304512, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:29:23.989574Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 89521828352, + "allocated_bytes_peak_delta": 54055492096, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:28:15.713269Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8394673152, + "inactive_split_bytes_current_delta": 8261854208, + "inactive_split_bytes_peak": 11829742592, + "inactive_split_bytes_peak_delta": 11669356544, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 99994304512, + "reserved_bytes_current_delta": 60750299136, + "reserved_bytes_peak": 99994304512, + "reserved_bytes_peak_delta": 60750299136, + "segment_count_current": 421, + "segment_count_current_delta": 350, + "segment_count_peak": 421, + "segment_count_peak_delta": 350, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T142923990193Z.pickle", + "timestamp_utc": "2026-08-01T14:29:24.110909Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T142923990193Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T142923990193Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f1126b4849d7db0ce5bf4265775cfb2f4ba5421 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T142923990193Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e236ffe8e63a19060a6866b1c30dc7debaf40cd9082c1ac557c7ba788a80132a +size 5956779 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T142925433532Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T142925433532Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e9e3cb9120617a12cd91667a2269d8bb8f7d674a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T142925433532Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113852743680, + "active_bytes_peak_delta": 23978593792, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113852743680, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156510208, + "device": 2, + "inactive_split_bytes_current": 12568005632, + "inactive_split_bytes_peak": 13066544128, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:25.433104Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156510208, + "allocated_bytes_peak_delta": 23634681856, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 89874149888, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 89521828352, + "device": 2, + "inactive_split_bytes_current": 8394673152, + "inactive_split_bytes_peak": 11829742592, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 99994304512, + "reserved_bytes_peak": 99994304512, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:29:24.126243Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12568005632, + "inactive_split_bytes_current_delta": 4173332480, + "inactive_split_bytes_peak": 13066544128, + "inactive_split_bytes_peak_delta": 1236801536, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120835801088, + "reserved_bytes_current_delta": 20841496576, + "reserved_bytes_peak": 120835801088, + "reserved_bytes_peak_delta": 20841496576, + "segment_count_current": 485, + "segment_count_current_delta": 64, + "segment_count_peak": 485, + "segment_count_peak_delta": 64, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T142925433532Z.pickle", + "timestamp_utc": "2026-08-01T14:29:25.548526Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T142925433532Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T142925433532Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d2a442c07d2b2aaf8693b80353b46f4c1eb57e6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T142925433532Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7548e80fcc7a9e7a9e6edc43f5182610d157efc45f7bd78740e09836f6038607 +size 6101767 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T142926766762Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T142926766762Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ad76289bf89ad85a944e3b39164a7debebad0b53 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T142926766762Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113861788672, + "active_bytes_peak_delta": 9044992, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113861788672, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157145600, + "device": 2, + "inactive_split_bytes_current": 11215342592, + "inactive_split_bytes_peak": 13563214336, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:26.766344Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157145600, + "allocated_bytes_peak_delta": 635392, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113852743680, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113156510208, + "device": 2, + "inactive_split_bytes_current": 12568005632, + "inactive_split_bytes_peak": 13066544128, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:25.561856Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 11215342592, + "inactive_split_bytes_current_delta": -1352663040, + "inactive_split_bytes_peak": 13563214336, + "inactive_split_bytes_peak_delta": 496670208, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120835801088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120835801088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T142926766762Z.pickle", + "timestamp_utc": "2026-08-01T14:29:26.873426Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T142926766762Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T142926766762Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bfcd3f4f2194576912fc6eeb119df74339be5809 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T142926766762Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39e30a8d896b34a4ff94a0a51cb9a54ecc3daf3208a15e2c43897427c528dd7a +size 5952389 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T142928139900Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T142928139900Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f551ef70781317ec8cc86d44c109fe46235746d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T142928139900Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113861788672, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113861788672, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157145600, + "device": 2, + "inactive_split_bytes_current": 12154866688, + "inactive_split_bytes_peak": 13563214336, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:28.139496Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157145600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113861788672, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157145600, + "device": 2, + "inactive_split_bytes_current": 11215342592, + "inactive_split_bytes_peak": 13563214336, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:26.885930Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 12154866688, + "inactive_split_bytes_current_delta": 939524096, + "inactive_split_bytes_peak": 13563214336, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120835801088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120835801088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T142928139900Z.pickle", + "timestamp_utc": "2026-08-01T14:29:28.245096Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T142928139900Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T142928139900Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b1008556b5361c7bdaf509a0923993b9e1d059c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T142928139900Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb37564182d3f2a72715b56f48cd8e95d640b869e04fe6d7517f3d6079ceab73 +size 5952005 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T142929737210Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T142929737210Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5f90e10e8c3683bd9f3620c65f96229bc3284908 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T142929737210Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113861788672, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113861788672, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157145600, + "device": 2, + "inactive_split_bytes_current": 11213245440, + "inactive_split_bytes_peak": 13807970816, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:29.736782Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157145600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113861788672, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157145600, + "device": 2, + "inactive_split_bytes_current": 12154866688, + "inactive_split_bytes_peak": 13563214336, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:28.257329Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 11213245440, + "inactive_split_bytes_current_delta": -941621248, + "inactive_split_bytes_peak": 13807970816, + "inactive_split_bytes_peak_delta": 244756480, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120835801088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120835801088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T142929737210Z.pickle", + "timestamp_utc": "2026-08-01T14:29:29.843973Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T142929737210Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T142929737210Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..861f4823b83a68057aaa2e0c8ab9b002d779db4a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T142929737210Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6826d9892f902669cf163163012598e21e7517df08aa778d7a9dce46339a7d15 +size 5952018 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T142931061269Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T142931061269Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f424909cd5b70f554707bdd69edf9f6782de367 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T142931061269Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113861788672, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113861788672, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157145600, + "device": 2, + "inactive_split_bytes_current": 11567664128, + "inactive_split_bytes_peak": 13807970816, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:31.060843Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157145600, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113861788672, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157145600, + "device": 2, + "inactive_split_bytes_current": 11213245440, + "inactive_split_bytes_peak": 13807970816, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:29.856747Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 11567664128, + "inactive_split_bytes_current_delta": 354418688, + "inactive_split_bytes_peak": 13807970816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120835801088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120835801088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T142931061269Z.pickle", + "timestamp_utc": "2026-08-01T14:29:31.165730Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T142931061269Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T142931061269Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14c1725e05ce626d936ce0e965c883e8114f1922 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T142931061269Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:360a65b88281209dc0610070cd88474515ab709c11966a898c0299f8b56a83bc +size 6037825 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T142932381095Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T142932381095Z.json new file mode 100644 index 0000000000000000000000000000000000000000..79acb203c428da8947753caeb3fcf035b8a5aae6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T142932381095Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113862050304, + "active_bytes_peak_delta": 261632, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113862050304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157407232, + "device": 2, + "inactive_split_bytes_current": 10217098240, + "inactive_split_bytes_peak": 13807970816, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:32.380698Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157407232, + "allocated_bytes_peak_delta": 261632, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113861788672, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157145600, + "device": 2, + "inactive_split_bytes_current": 11567664128, + "inactive_split_bytes_peak": 13807970816, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:31.178430Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10217098240, + "inactive_split_bytes_current_delta": -1350565888, + "inactive_split_bytes_peak": 13807970816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120835801088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120835801088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T142932381095Z.pickle", + "timestamp_utc": "2026-08-01T14:29:32.487831Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T142932381095Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T142932381095Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6749381fd427cdf2647d0d0a42c8f1725912de25 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T142932381095Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a66bcf04278031a9ec377f2855d815903149969dd711b1590ad157217b955771 +size 5995089 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T142933937766Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T142933937766Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4c51d17e34fb57d3077bdc3731d0ee3d2c9010d1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T142933937766Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113862050304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113862050304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157407232, + "device": 2, + "inactive_split_bytes_current": 11569761280, + "inactive_split_bytes_peak": 13807970816, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:33.937319Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157407232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113862050304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157407232, + "device": 2, + "inactive_split_bytes_current": 10217098240, + "inactive_split_bytes_peak": 13807970816, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:32.502128Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 11569761280, + "inactive_split_bytes_current_delta": 1352663040, + "inactive_split_bytes_peak": 13807970816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120835801088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120835801088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T142933937766Z.pickle", + "timestamp_utc": "2026-08-01T14:29:34.045211Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T142933937766Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T142933937766Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..90909039e9971f569398940fbfbc07880ab59da2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T142933937766Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:092fed19a1d9f35a4797b16741e87f4a47979e90f667f4c443333d5ecc2cdf91 +size 5951809 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T142935269620Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T142935269620Z.json new file mode 100644 index 0000000000000000000000000000000000000000..897d38e640cf5932ecb7f3415eccffede4f901b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T142935269620Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113862050304, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113862050304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157407232, + "device": 2, + "inactive_split_bytes_current": 10984655872, + "inactive_split_bytes_peak": 13807970816, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:35.269212Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113157407232, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113862050304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157407232, + "device": 2, + "inactive_split_bytes_current": 11569761280, + "inactive_split_bytes_peak": 13807970816, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:34.057220Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10984655872, + "inactive_split_bytes_current_delta": -585105408, + "inactive_split_bytes_peak": 13807970816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120835801088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120835801088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T142935269620Z.pickle", + "timestamp_utc": "2026-08-01T14:29:35.371214Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T142935269620Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T142935269620Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cbc5d58b236d4ef4e32ea1d460e89b2317be9138 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T142935269620Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:812ac4355bfb98df4712eb470e6e8a9a34d97dc84e29d90a8f255206cacd26cc +size 5994994 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T143034050741Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T143034050741Z.json new file mode 100644 index 0000000000000000000000000000000000000000..81dad72f93144fe37163c02c8e4030446c237a13 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T143034050741Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113862071808, + "active_bytes_peak_delta": 21504, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 113862071808, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 113157428736, + "device": 2, + "inactive_split_bytes_current": 10682665984, + "inactive_split_bytes_peak": 13807970816, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:30:34.050250Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113157428736, + "allocated_bytes_peak_delta": 21504, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113862050304, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113157407232, + "device": 2, + "inactive_split_bytes_current": 10984655872, + "inactive_split_bytes_peak": 13807970816, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120835801088, + "reserved_bytes_peak": 120835801088, + "segment_count_current": 485, + "segment_count_peak": 485, + "timestamp_utc": "2026-08-01T14:29:35.382978Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10682665984, + "inactive_split_bytes_current_delta": -301989888, + "inactive_split_bytes_peak": 13807970816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120835801088, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120835801088, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 485, + "segment_count_current_delta": 0, + "segment_count_peak": 485, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T143034050741Z.pickle", + "timestamp_utc": "2026-08-01T14:30:34.250277Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T143034050741Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T143034050741Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d40e9073921ce52fe2ecd70dee41a18e8374385c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T143034050741Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6b4ebb158e22b72597e5f0a456443afd91ec80facecdcb2236b27d96fead9ec +size 9741780 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T142923990248Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T142923990248Z.json new file mode 100644 index 0000000000000000000000000000000000000000..74fb667dd59589d9443f94283da86b0b11d5779e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T142923990248Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 90220190720, + "active_bytes_peak_delta": 54753854464, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 90220190720, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 89522089984, + "device": 3, + "inactive_split_bytes_current": 8335952896, + "inactive_split_bytes_peak": 12396105728, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 99528736768, + "reserved_bytes_peak": 99528736768, + "segment_count_current": 416, + "segment_count_peak": 416, + "timestamp_utc": "2026-08-01T14:29:23.989623Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 89522089984, + "allocated_bytes_peak_delta": 54055753728, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:28:15.713395Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8335952896, + "inactive_split_bytes_current_delta": 8203133952, + "inactive_split_bytes_peak": 12396105728, + "inactive_split_bytes_peak_delta": 12235719680, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 99528736768, + "reserved_bytes_current_delta": 60284731392, + "reserved_bytes_peak": 99528736768, + "reserved_bytes_peak_delta": 60284731392, + "segment_count_current": 416, + "segment_count_current_delta": 345, + "segment_count_peak": 416, + "segment_count_peak_delta": 345, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T142923990248Z.pickle", + "timestamp_utc": "2026-08-01T14:29:24.107256Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T142923990248Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T142923990248Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29defb2eeb66062f7cdf4bbdd193cdcf1a0b0391 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T142923990248Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec817916ed642a86f75af052a391436c24f71e49a7fa8ac78f94c06d3424193d +size 5938118 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T142925433612Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T142925433612Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3236e042080389e821bcca0fbd4c9609ed514835 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T142925433612Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113516131840, + "active_bytes_peak_delta": 23295941120, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113516131840, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113163810304, + "device": 3, + "inactive_split_bytes_current": 10863021056, + "inactive_split_bytes_peak": 13208365056, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:25.433188Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113163810304, + "allocated_bytes_peak_delta": 23641720320, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 90220190720, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 89522089984, + "device": 3, + "inactive_split_bytes_current": 8335952896, + "inactive_split_bytes_peak": 12396105728, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 99528736768, + "reserved_bytes_peak": 99528736768, + "segment_count_current": 416, + "segment_count_peak": 416, + "timestamp_utc": "2026-08-01T14:29:24.126592Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10863021056, + "inactive_split_bytes_current_delta": 2527068160, + "inactive_split_bytes_peak": 13208365056, + "inactive_split_bytes_peak_delta": 812259328, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120659640320, + "reserved_bytes_current_delta": 21130903552, + "reserved_bytes_peak": 120659640320, + "reserved_bytes_peak_delta": 21130903552, + "segment_count_current": 483, + "segment_count_current_delta": 67, + "segment_count_peak": 483, + "segment_count_peak_delta": 67, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T142925433612Z.pickle", + "timestamp_utc": "2026-08-01T14:29:25.550637Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T142925433612Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T142925433612Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ef8b47ed198365b1d94bbb7705199d8a3d5f32f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T142925433612Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e2f50a10790a8aded97a9576fddffb94e566b899ae8e08fb4bb3a176ed88516 +size 5955642 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T142926766793Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T142926766793Z.json new file mode 100644 index 0000000000000000000000000000000000000000..807918f3e78161d60f2a124984009dc3870a6c39 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T142926766793Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113859822592, + "active_bytes_peak_delta": 343690752, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113859822592, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113163810304, + "device": 3, + "inactive_split_bytes_current": 10718317568, + "inactive_split_bytes_peak": 14058948096, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:26.766390Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113163810304, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113516131840, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113163810304, + "device": 3, + "inactive_split_bytes_current": 10863021056, + "inactive_split_bytes_peak": 13208365056, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:25.567153Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10718317568, + "inactive_split_bytes_current_delta": -144703488, + "inactive_split_bytes_peak": 14058948096, + "inactive_split_bytes_peak_delta": 850583040, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120659640320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120659640320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T142926766793Z.pickle", + "timestamp_utc": "2026-08-01T14:29:26.878264Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T142926766793Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T142926766793Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..672d999f28f6af5234c34173531b4ee97c243efe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T142926766793Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4107190dfcccf6eee8c4fd706dfbe34be080dcc513e8090aef332f8562d05cb +size 6037275 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T142928139948Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T142928139948Z.json new file mode 100644 index 0000000000000000000000000000000000000000..488066788cd07117a79e1556ae3c493d3a105046 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T142928139948Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113870549504, + "active_bytes_peak_delta": 10726912, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 9717976064, + "inactive_split_bytes_peak": 14058948096, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:28.139544Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165906432, + "allocated_bytes_peak_delta": 2096128, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113859822592, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113163810304, + "device": 3, + "inactive_split_bytes_current": 10718317568, + "inactive_split_bytes_peak": 14058948096, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:26.894511Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9717976064, + "inactive_split_bytes_current_delta": -1000341504, + "inactive_split_bytes_peak": 14058948096, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120659640320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120659640320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T142928139948Z.pickle", + "timestamp_utc": "2026-08-01T14:29:28.251248Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T142928139948Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T142928139948Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..431ca543fff671aead65cc9b9564fc593c40349e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T142928139948Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aca3b126429b432c6d72003b06e8d8a866b03afb29eeaeb2873f6c37275b9a5c +size 5951038 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T142929737200Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T142929737200Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ea365a719d211b05a59c96c59fd519466ac0da5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T142929737200Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113870549504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 10571516928, + "inactive_split_bytes_peak": 15347446272, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:29.736800Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165906432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 9717976064, + "inactive_split_bytes_peak": 14058948096, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:28.266967Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10571516928, + "inactive_split_bytes_current_delta": 853540864, + "inactive_split_bytes_peak": 15347446272, + "inactive_split_bytes_peak_delta": 1288498176, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120659640320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120659640320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T142929737200Z.pickle", + "timestamp_utc": "2026-08-01T14:29:29.847476Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T142929737200Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T142929737200Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd79b83b3c1a1a7239f8878b9f336981bd4ff298 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T142929737200Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff14a91728aee08933b21634d553993ccd6c85e75b9da3b0bfa8f40f8158bd88 +size 5950784 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T142931061297Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T142931061297Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c07a4bc148b679f9816ddc7c98487d785c9ee7ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T142931061297Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113870549504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 9808153600, + "inactive_split_bytes_peak": 15347446272, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:31.060887Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165906432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 10571516928, + "inactive_split_bytes_peak": 15347446272, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:29.863333Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9808153600, + "inactive_split_bytes_current_delta": -763363328, + "inactive_split_bytes_peak": 15347446272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120659640320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120659640320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T142931061297Z.pickle", + "timestamp_utc": "2026-08-01T14:29:31.170422Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T142931061297Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T142931061297Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aa95b895dd07025e87dd8754505508eb01514fa8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T142931061297Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c970334ce82445481a772415c058710adc4bb90dd325601c47efc1ae1c67fd0c +size 5993887 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T142932381136Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T142932381136Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7480057a9532ba4fb14010981af53c7575fd40b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T142932381136Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113870549504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 10688957440, + "inactive_split_bytes_peak": 15347446272, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:32.380741Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165906432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 9808153600, + "inactive_split_bytes_peak": 15347446272, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:31.186188Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10688957440, + "inactive_split_bytes_current_delta": 880803840, + "inactive_split_bytes_peak": 15347446272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120659640320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120659640320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T142932381136Z.pickle", + "timestamp_utc": "2026-08-01T14:29:32.490555Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T142932381136Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T142932381136Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..394323d3901f9c87d51a59e2d7348df43667c268 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T142932381136Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ba95700643c6d812ef256c6df8aa4784fad280b98217796a3dd1b01babdffd6 +size 6036260 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T142933937811Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T142933937811Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15e792b719fcf8871be5cf06c98aedd49de1d847 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T142933937811Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113870549504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 11158719488, + "inactive_split_bytes_peak": 15347446272, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:33.937397Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165906432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 10688957440, + "inactive_split_bytes_peak": 15347446272, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:32.507066Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 11158719488, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 15347446272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120659640320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120659640320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T142933937811Z.pickle", + "timestamp_utc": "2026-08-01T14:29:34.053308Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T142933937811Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T142933937811Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..84bb992a91af5bf6f113e48c44b1cd28752a6b70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T142933937811Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc876aa9720dd830db3bd40a1178ea1e289bf883f742b73f9627b2bfc8dadb67 +size 6036298 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T142935269637Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T142935269637Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c0392eeb3c90675a28d835321e9a1718f38718d4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T142935269637Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113870549504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 10804300800, + "inactive_split_bytes_peak": 15347446272, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:35.269241Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113165906432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 11158719488, + "inactive_split_bytes_peak": 15347446272, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:34.073505Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10804300800, + "inactive_split_bytes_current_delta": -354418688, + "inactive_split_bytes_peak": 15347446272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120659640320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120659640320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T142935269637Z.pickle", + "timestamp_utc": "2026-08-01T14:29:35.378346Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T142935269637Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T142935269637Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..db094d2902a3ae3a27151f634695eb69053fcbc1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T142935269637Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a8ddccb7728460e339c7fb3df5a39ad287516e14574048114d38db059983103 +size 5950463 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T143034050729Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T143034050729Z.json new file mode 100644 index 0000000000000000000000000000000000000000..32ad2547d2c096d6001225e279fa74510782737e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T143034050729Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113870549504, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 12094049280, + "inactive_split_bytes_peak": 15347446272, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:30:34.050259Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113165906432, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113870549504, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113165906432, + "device": 3, + "inactive_split_bytes_current": 10804300800, + "inactive_split_bytes_peak": 15347446272, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 120659640320, + "reserved_bytes_peak": 120659640320, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:35.394702Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 12094049280, + "inactive_split_bytes_current_delta": 1289748480, + "inactive_split_bytes_peak": 15347446272, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120659640320, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120659640320, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T143034050729Z.pickle", + "timestamp_utc": "2026-08-01T14:30:34.251354Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T143034050729Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T143034050729Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8ba5781a50365f81aaaac11d033fe5da7993e343 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T143034050729Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd2e8de22d3428c8ed258c85631efc7f34dbaf6647b53e5297fa6797c5b50965 +size 9740879 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T142923990109Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T142923990109Z.json new file mode 100644 index 0000000000000000000000000000000000000000..85ef80af28eec2d3381789ffcbb68e33faee904c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T142923990109Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441682944, + "active_bytes_current_delta": 37209785344, + "active_bytes_peak": 89872182784, + "active_bytes_peak_delta": 54405404160, + "after": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 89872182784, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 89519861248, + "device": 4, + "inactive_split_bytes_current": 8879580160, + "inactive_split_bytes_peak": 12609593344, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 99881058304, + "reserved_bytes_peak": 99881058304, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T14:29:23.989534Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441682944, + "allocated_bytes_current_delta": 37209785344, + "allocated_bytes_peak": 89519861248, + "allocated_bytes_peak_delta": 54053082624, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:28:15.712404Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8879580160, + "inactive_split_bytes_current_delta": 8747203584, + "inactive_split_bytes_peak": 12609593344, + "inactive_split_bytes_peak_delta": 12456183808, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 99881058304, + "reserved_bytes_current_delta": 60637052928, + "reserved_bytes_peak": 99881058304, + "reserved_bytes_peak_delta": 60637052928, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T142923990109Z.pickle", + "timestamp_utc": "2026-08-01T14:29:24.108661Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T142923990109Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T142923990109Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c75ed24f5523ff4f1405e98ef4914b9da0b948ef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T142923990109Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5daad410e5aed54fdc2fe917b12835bcd6ef94d4c2754de427d92125ff228747 +size 7020411 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T142925433620Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T142925433620Z.json new file mode 100644 index 0000000000000000000000000000000000000000..819a46c6be07a4a1982dc4e702e8bf906419f1b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T142925433620Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441682944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113508595712, + "active_bytes_peak_delta": 23636412928, + "after": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113508595712, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 10643284992, + "inactive_split_bytes_peak": 14184534528, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120156323840, + "reserved_bytes_peak": 120156323840, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:25.433188Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441682944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156274176, + "allocated_bytes_peak_delta": 23636412928, + "before": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 89872182784, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 89519861248, + "device": 4, + "inactive_split_bytes_current": 8879580160, + "inactive_split_bytes_peak": 12609593344, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 99881058304, + "reserved_bytes_peak": 99881058304, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T14:29:24.127981Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10643284992, + "inactive_split_bytes_current_delta": 1763704832, + "inactive_split_bytes_peak": 14184534528, + "inactive_split_bytes_peak_delta": 1574941184, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120156323840, + "reserved_bytes_current_delta": 20275265536, + "reserved_bytes_peak": 120156323840, + "reserved_bytes_peak_delta": 20275265536, + "segment_count_current": 483, + "segment_count_current_delta": 63, + "segment_count_peak": 483, + "segment_count_peak_delta": 63, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T142925433620Z.pickle", + "timestamp_utc": "2026-08-01T14:29:25.551083Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T142925433620Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T142925433620Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c152a87e476cd340b9552fa383eff9ee3bff17d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T142925433620Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84f643367d4bfdcf340c6568aa77e56434a89dfbccc43223a43ef431a4283b7e +size 5977574 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T142926766887Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T142926766887Z.json new file mode 100644 index 0000000000000000000000000000000000000000..40a9287600813f32fd08f780ebcbf2780c0cb3bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T142926766887Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441682944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113856668160, + "active_bytes_peak_delta": 348072448, + "after": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113856668160, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 10878166016, + "inactive_split_bytes_peak": 15238713856, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:26.766456Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441682944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156274176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113508595712, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 10643284992, + "inactive_split_bytes_peak": 14184534528, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120156323840, + "reserved_bytes_peak": 120156323840, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:25.567516Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10878166016, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 15238713856, + "inactive_split_bytes_peak_delta": 1054179328, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120980504576, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 120980504576, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 484, + "segment_count_current_delta": 1, + "segment_count_peak": 484, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T142926766887Z.pickle", + "timestamp_utc": "2026-08-01T14:29:26.877088Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T142926766887Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T142926766887Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5b843a7bd72c9a9d5665402b5d0691a5606ee26d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T142926766887Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b07871a2c37f6fa7a3cfd780c772a58e5e9bb85226f1405d6ba803504a2be693 +size 5973980 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T142928139925Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T142928139925Z.json new file mode 100644 index 0000000000000000000000000000000000000000..83f6569aeae0536e8163624ae4099f379de0e859 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T142928139925Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441682944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113856668160, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113856668160, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 11173864448, + "inactive_split_bytes_peak": 15238713856, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:28.139501Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441682944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156274176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113856668160, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 10878166016, + "inactive_split_bytes_peak": 15238713856, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:26.893147Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11173864448, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 15238713856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120980504576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120980504576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 484, + "segment_count_current_delta": 0, + "segment_count_peak": 484, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T142928139925Z.pickle", + "timestamp_utc": "2026-08-01T14:29:28.249376Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T142928139925Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T142928139925Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b23d05636b319c72e85abffdd53842cd4d89a2ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T142928139925Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d7cd7451e92073d941ce2b7a6723c0575feaced78aa5c3ea9c3bdde3be25814 +size 5974620 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T142929737175Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T142929737175Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a399c9433c51bcc1e1848b171086a255a5a9d8a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T142929737175Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441682944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113858294272, + "active_bytes_peak_delta": 1626112, + "after": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113858294272, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 10525844480, + "inactive_split_bytes_peak": 15238713856, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:29.736755Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441682944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156274176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113856668160, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 11173864448, + "inactive_split_bytes_peak": 15238713856, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:28.264983Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10525844480, + "inactive_split_bytes_current_delta": -648019968, + "inactive_split_bytes_peak": 15238713856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120980504576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120980504576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 484, + "segment_count_current_delta": 0, + "segment_count_peak": 484, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T142929737175Z.pickle", + "timestamp_utc": "2026-08-01T14:29:29.846533Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T142929737175Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T142929737175Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e0d3577a655081437d22477eed2e9cd377ba09be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T142929737175Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f931da2ba6827e08d87c351b2411f4191277573b6d990f4580cd66b3fc10cc5d +size 5976981 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T142931061318Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T142931061318Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b8d4736fe6cd1dfad14f0719ef974a29dabe4a3d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T142931061318Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441682944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113859868672, + "active_bytes_peak_delta": 1574400, + "after": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113859868672, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 11526185984, + "inactive_split_bytes_peak": 15238713856, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:31.060906Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441682944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156274176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113858294272, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 10525844480, + "inactive_split_bytes_peak": 15238713856, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:29.862265Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11526185984, + "inactive_split_bytes_current_delta": 1000341504, + "inactive_split_bytes_peak": 15238713856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120980504576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120980504576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 484, + "segment_count_current_delta": 0, + "segment_count_peak": 484, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T142931061318Z.pickle", + "timestamp_utc": "2026-08-01T14:29:31.175475Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T142931061318Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T142931061318Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d08cf39a5e21647778362ad2f6ff6c2c28cf25e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T142931061318Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01926a12667b3518653f7f26765b1b2aaeb13a9fe77fbfacf195be4cb4fded84 +size 5976862 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T142932381137Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T142932381137Z.json new file mode 100644 index 0000000000000000000000000000000000000000..50e27e6474963e4b2fd8cc82de01c5fa64bf7eb1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T142932381137Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441682944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113859868672, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113859868672, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 11905770496, + "inactive_split_bytes_peak": 15238713856, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:32.380735Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441682944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156274176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113859868672, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 11526185984, + "inactive_split_bytes_peak": 15238713856, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:31.193947Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11905770496, + "inactive_split_bytes_current_delta": 379584512, + "inactive_split_bytes_peak": 15238713856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120980504576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120980504576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 484, + "segment_count_current_delta": 0, + "segment_count_peak": 484, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T142932381137Z.pickle", + "timestamp_utc": "2026-08-01T14:29:32.488962Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T142932381137Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T142932381137Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f7faf3f2b074bceb79187db0b91c7be7773c156c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T142932381137Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d590eefc865724e9a43cff5625a8bc9300ca294be059f1ffc132a2c7c9c211b +size 5976577 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T142933937773Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T142933937773Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06d2f20f279a4d4549e03aed49d39b190e84b9a6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T142933937773Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441682944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113859868672, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113859868672, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 10821542912, + "inactive_split_bytes_peak": 15238713856, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:33.937325Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441682944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156274176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113859868672, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 11905770496, + "inactive_split_bytes_peak": 15238713856, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:32.505882Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10821542912, + "inactive_split_bytes_current_delta": -1084227584, + "inactive_split_bytes_peak": 15238713856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120980504576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120980504576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 484, + "segment_count_current_delta": 0, + "segment_count_peak": 484, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T142933937773Z.pickle", + "timestamp_utc": "2026-08-01T14:29:34.047489Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T142933937773Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T142933937773Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6d6c23885f4de309cec6007a4d22f29ea2d9dba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T142933937773Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f6804e9bb1ee31b33084f8a37e0245d4473d3daa87c39cb6ca770593c2a5010 +size 5976446 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T142935269600Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T142935269600Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ce381b24223a42c0f2581cadfc9296e769f7b1d6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T142935269600Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441682944, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113859868672, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113859868672, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 10643284992, + "inactive_split_bytes_peak": 15238713856, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:35.269210Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441682944, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156274176, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113859868672, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 10821542912, + "inactive_split_bytes_peak": 15238713856, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:34.064000Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 10643284992, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 15238713856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120980504576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120980504576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 484, + "segment_count_current_delta": 0, + "segment_count_peak": 484, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T142935269600Z.pickle", + "timestamp_utc": "2026-08-01T14:29:35.376817Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T142935269600Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T142935269600Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..864a079b0dc3268d55711ad86312842e5dfc5890 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T142935269600Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30ec5ccd2b9944ed0b4045801fb22ac3e2d0e71f702d7d3068f6588cb2f45a5a +size 5976608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T143034050726Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T143034050726Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1160dd0109814b411a98cb371e753fc30bfc511f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T143034050726Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61167393792, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113867796992, + "active_bytes_peak_delta": 7928320, + "after": { + "active_bytes_current": 61167393792, + "active_bytes_peak": 113867796992, + "allocated_bytes_current": 61167393792, + "allocated_bytes_peak": 113163153920, + "device": 4, + "inactive_split_bytes_current": 11641529344, + "inactive_split_bytes_peak": 15238713856, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:30:34.050256Z", + "world_size": 8 + }, + "allocated_bytes_current": 61167393792, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113163153920, + "allocated_bytes_peak_delta": 6879744, + "before": { + "active_bytes_current": 72441682944, + "active_bytes_peak": 113859868672, + "allocated_bytes_current": 72441682944, + "allocated_bytes_peak": 113156274176, + "device": 4, + "inactive_split_bytes_current": 10643284992, + "inactive_split_bytes_peak": 15238713856, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120980504576, + "reserved_bytes_peak": 120980504576, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:35.393126Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 11641529344, + "inactive_split_bytes_current_delta": 998244352, + "inactive_split_bytes_peak": 15238713856, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120980504576, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120980504576, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 484, + "segment_count_current_delta": 0, + "segment_count_peak": 484, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T143034050726Z.pickle", + "timestamp_utc": "2026-08-01T14:30:34.252303Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T143034050726Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T143034050726Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e90b86677da5cc543d175d31ec49a576e9d2279 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T143034050726Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e8b932759ee8331a3fbda0d7f3dac19f1fe9622597a85b2589b1dbcb5e96bb7 +size 9781256 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T142923990136Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T142923990136Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3caf545e9b2e8c5ad63779781934c1ac2c0824c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T142923990136Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 89880572928, + "active_bytes_peak_delta": 54414236672, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 89880572928, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 89528251392, + "device": 5, + "inactive_split_bytes_current": 8805714944, + "inactive_split_bytes_peak": 13090524160, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 100287905792, + "reserved_bytes_peak": 100287905792, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T14:29:23.989533Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 89528251392, + "allocated_bytes_peak_delta": 54061915136, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:28:15.712962Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8805714944, + "inactive_split_bytes_current_delta": 8672896000, + "inactive_split_bytes_peak": 13090524160, + "inactive_split_bytes_peak_delta": 12930138112, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 100287905792, + "reserved_bytes_current_delta": 61043900416, + "reserved_bytes_peak": 100287905792, + "reserved_bytes_peak_delta": 61043900416, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T142923990136Z.pickle", + "timestamp_utc": "2026-08-01T14:29:24.108946Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T142923990136Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T142923990136Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..087cfd4e18bb6b2c6f99bab95360c28da2f3e062 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T142923990136Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b20aec8d7457c9c00514192ed8de481a74add9fe565713bc88dd0aa29e92215c +size 5941168 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T142925433567Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T142925433567Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87c75266a43b41881afd2562d337c3bb55f09149 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T142925433567Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113507631616, + "active_bytes_peak_delta": 23627058688, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113507631616, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113155310080, + "device": 5, + "inactive_split_bytes_current": 10596682752, + "inactive_split_bytes_peak": 14244506624, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:25.433117Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113155310080, + "allocated_bytes_peak_delta": 23627058688, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 89880572928, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 89528251392, + "device": 5, + "inactive_split_bytes_current": 8805714944, + "inactive_split_bytes_peak": 13090524160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 100287905792, + "reserved_bytes_peak": 100287905792, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T14:29:24.127974Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10596682752, + "inactive_split_bytes_current_delta": 1790967808, + "inactive_split_bytes_peak": 14244506624, + "inactive_split_bytes_peak_delta": 1153982464, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120330387456, + "reserved_bytes_current_delta": 20042481664, + "reserved_bytes_peak": 120330387456, + "reserved_bytes_peak_delta": 20042481664, + "segment_count_current": 486, + "segment_count_current_delta": 66, + "segment_count_peak": 486, + "segment_count_peak_delta": 66, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T142925433567Z.pickle", + "timestamp_utc": "2026-08-01T14:29:25.551529Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T142925433567Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T142925433567Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2de9173abcf4cfd5c199c2f56ec5e27d02afb3d7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T142925433567Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62b07be815b1457834656e3d4d803ae55945d68ebb95992def9822089d254786 +size 6083848 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T142926766753Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T142926766753Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dfdbcd60b623c433f70ec3d6dec4c6ad63c2d8f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T142926766753Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113866216960, + "active_bytes_peak_delta": 358585344, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 10447784960, + "inactive_split_bytes_peak": 15094586880, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:26.766340Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113161573888, + "allocated_bytes_peak_delta": 6263808, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113507631616, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113155310080, + "device": 5, + "inactive_split_bytes_current": 10596682752, + "inactive_split_bytes_peak": 14244506624, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:25.567863Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10447784960, + "inactive_split_bytes_current_delta": -148897792, + "inactive_split_bytes_peak": 15094586880, + "inactive_split_bytes_peak_delta": 850080256, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120330387456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120330387456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T142926766753Z.pickle", + "timestamp_utc": "2026-08-01T14:29:26.877510Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T142926766753Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T142926766753Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ecbe7541147670b9940e5d739d11866e599ba72c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T142926766753Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8be10fdf103aee82500f7a57e5d0acd970279bf5d9e61ee2ede78ebf953e603f +size 5952506 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T142928139902Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T142928139902Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea42c0eac1a1b51d144e8058b6b624a92df1a4e3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T142928139902Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113866216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 9862679552, + "inactive_split_bytes_peak": 15094586880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:28.139493Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113161573888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 10447784960, + "inactive_split_bytes_peak": 15094586880, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:26.893493Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9862679552, + "inactive_split_bytes_current_delta": -585105408, + "inactive_split_bytes_peak": 15094586880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120330387456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120330387456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T142928139902Z.pickle", + "timestamp_utc": "2026-08-01T14:29:28.249584Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T142928139902Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T142928139902Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b656620f21124024617bb4b7d64ba20a0cd016bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T142928139902Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e83c2f5d0efec41a3f02d293395864fb52a0a1854b282a31075b886d77263184 +size 5952228 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T142929737257Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T142929737257Z.json new file mode 100644 index 0000000000000000000000000000000000000000..655afbe0c118a0ca560e5bec71bf982c5642d257 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T142929737257Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113866216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 10273721344, + "inactive_split_bytes_peak": 15094586880, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:29.736820Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113161573888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 9862679552, + "inactive_split_bytes_peak": 15094586880, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:28.265187Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10273721344, + "inactive_split_bytes_current_delta": 411041792, + "inactive_split_bytes_peak": 15094586880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120330387456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120330387456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T142929737257Z.pickle", + "timestamp_utc": "2026-08-01T14:29:29.847908Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T142929737257Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T142929737257Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d80745764bdffc37c5b861690454f7d30e15aeb1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T142929737257Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0ade8b684bdf56833ea7dfea852cd9fe00e8a818ce8dc86f180a07058d3e180 +size 5995149 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T142931061292Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T142931061292Z.json new file mode 100644 index 0000000000000000000000000000000000000000..40a6ae54188f6d5f2dbf14e6184e9203db686440 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T142931061292Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113866216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 10273721344, + "inactive_split_bytes_peak": 15094586880, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:31.060856Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113161573888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 10273721344, + "inactive_split_bytes_peak": 15094586880, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:29.863583Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10273721344, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 15094586880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120330387456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120330387456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T142931061292Z.pickle", + "timestamp_utc": "2026-08-01T14:29:31.168896Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T142931061292Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T142931061292Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c1daa2b394d36884ec053869ee2c439dddba748 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T142931061292Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de9cd12c12064ed25d70dd149fa82c8067de647b5be7236ab6d5b2d942d32c23 +size 5951521 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T142932381129Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T142932381129Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04d437734969928ea2a4ef27559d0dfabfbd4ea5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T142932381129Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113866216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 10038840320, + "inactive_split_bytes_peak": 15094586880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:32.380709Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113161573888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 10273721344, + "inactive_split_bytes_peak": 15094586880, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:31.184297Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10038840320, + "inactive_split_bytes_current_delta": -234881024, + "inactive_split_bytes_peak": 15094586880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120330387456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120330387456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T142932381129Z.pickle", + "timestamp_utc": "2026-08-01T14:29:32.488887Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T142932381129Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T142932381129Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8e62bb7de6bf10cd511ca2ad60cbc01102edd122 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T142932381129Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:668ce33e7e9dd4fdf53184e0b7030b2a5911b157f465ac44f7a3c1091becfa3c +size 5951441 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T142933937724Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T142933937724Z.json new file mode 100644 index 0000000000000000000000000000000000000000..73b1fdab5f613670d54060f818c828fd1662cb92 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T142933937724Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113866216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 9862679552, + "inactive_split_bytes_peak": 15094586880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:33.937324Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113161573888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 10038840320, + "inactive_split_bytes_peak": 15094586880, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:32.505147Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9862679552, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 15094586880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120330387456, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120330387456, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T142933937724Z.pickle", + "timestamp_utc": "2026-08-01T14:29:34.047531Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T142933937724Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T142933937724Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..38c73857085855220ca80df16a5dfb69257ce654 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T142933937724Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fbf7992fed5e2a078cb34cce4f3645b64913b6c4282c63119470ae10e2094a9 +size 5951462 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T142935269591Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T142935269591Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e144d7e46de08710c28853d5b415976a84b051ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T142935269591Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113866216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 10158377984, + "inactive_split_bytes_peak": 15094586880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121154568192, + "reserved_bytes_peak": 121154568192, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T14:29:35.269199Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113161573888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 9862679552, + "inactive_split_bytes_peak": 15094586880, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120330387456, + "reserved_bytes_peak": 120330387456, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:34.063637Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10158377984, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 15094586880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121154568192, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 121154568192, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 487, + "segment_count_current_delta": 1, + "segment_count_peak": 487, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T142935269591Z.pickle", + "timestamp_utc": "2026-08-01T14:29:35.378267Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T142935269591Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T142935269591Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7eb2e0c53ec150603c531248191942d50f680102 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T142935269591Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94e22887163860abbf97cd87730f1cdc90484eaba491a943f158a741062c3e62 +size 6096398 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T143034050707Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T143034050707Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4d111af826d5030df7b75bde1051c02809bdcbeb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T143034050707Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113866216960, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 10212903936, + "inactive_split_bytes_peak": 15094586880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121154568192, + "reserved_bytes_peak": 121154568192, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T14:30:34.050244Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113161573888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113866216960, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113161573888, + "device": 5, + "inactive_split_bytes_current": 10158377984, + "inactive_split_bytes_peak": 15094586880, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121154568192, + "reserved_bytes_peak": 121154568192, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T14:29:35.394422Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10212903936, + "inactive_split_bytes_current_delta": 54525952, + "inactive_split_bytes_peak": 15094586880, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121154568192, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121154568192, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 487, + "segment_count_current_delta": 0, + "segment_count_peak": 487, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T143034050707Z.pickle", + "timestamp_utc": "2026-08-01T14:30:34.255674Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T143034050707Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T143034050707Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2f44efa7fcb0fe1fadc89f20b5df0d0cf9eff8f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T143034050707Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2afdb89fbaedffa43e49680fe1c76bdc01f6d693a5bc70e149adc378ec828b8 +size 9741403 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T142923990183Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T142923990183Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ba1200e7679672c3911a345a0a9fcbb2ef74ff6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T142923990183Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 90220190720, + "active_bytes_peak_delta": 54753854464, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 90220190720, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 89523007488, + "device": 6, + "inactive_split_bytes_current": 8335952896, + "inactive_split_bytes_peak": 11986243584, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 99998498816, + "reserved_bytes_peak": 99998498816, + "segment_count_current": 418, + "segment_count_peak": 418, + "timestamp_utc": "2026-08-01T14:29:23.989559Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 89523007488, + "allocated_bytes_peak_delta": 54056671232, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:28:15.712454Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8335952896, + "inactive_split_bytes_current_delta": 8203133952, + "inactive_split_bytes_peak": 11986243584, + "inactive_split_bytes_peak_delta": 11825857536, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 99998498816, + "reserved_bytes_current_delta": 60754493440, + "reserved_bytes_peak": 99998498816, + "reserved_bytes_peak_delta": 60754493440, + "segment_count_current": 418, + "segment_count_current_delta": 347, + "segment_count_peak": 418, + "segment_count_peak_delta": 347, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T142923990183Z.pickle", + "timestamp_utc": "2026-08-01T14:29:24.115321Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T142923990183Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T142923990183Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7cc35fc7a18af2d11bca58d05cbb4d80b85eaf1a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T142923990183Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb6f2454ef801e4d87d2881345c573b4a77c127795389d20165ddf03837905fc +size 5955871 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T142925433846Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T142925433846Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5cebe51675cfd880d8fea9203e594be5a44ebd66 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T142925433846Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113507860992, + "active_bytes_peak_delta": 23287670272, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113507860992, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113155572224, + "device": 6, + "inactive_split_bytes_current": 10271624192, + "inactive_split_bytes_peak": 13181936128, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:25.433313Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113155572224, + "allocated_bytes_peak_delta": 23632564736, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 90220190720, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 89523007488, + "device": 6, + "inactive_split_bytes_current": 8335952896, + "inactive_split_bytes_peak": 11986243584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 99998498816, + "reserved_bytes_peak": 99998498816, + "segment_count_current": 418, + "segment_count_peak": 418, + "timestamp_utc": "2026-08-01T14:29:24.135581Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10271624192, + "inactive_split_bytes_current_delta": 1935671296, + "inactive_split_bytes_peak": 13181936128, + "inactive_split_bytes_peak_delta": 1195692544, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 120747720704, + "reserved_bytes_current_delta": 20749221888, + "reserved_bytes_peak": 120747720704, + "reserved_bytes_peak_delta": 20749221888, + "segment_count_current": 486, + "segment_count_current_delta": 68, + "segment_count_peak": 486, + "segment_count_peak_delta": 68, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T142925433846Z.pickle", + "timestamp_utc": "2026-08-01T14:29:25.553805Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T142925433846Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T142925433846Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..831017e4d2e63d6ec3efda99652c84992e196171 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T142925433846Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc2ff2451d60e241ee7e3f38e791be1efd9c445a4021d8415889cab49d631cf4 +size 6107240 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T142926766976Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T142926766976Z.json new file mode 100644 index 0000000000000000000000000000000000000000..790b93ebe3fcf7e604364e1b90bee616ea65cef0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T142926766976Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113507860992, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113507860992, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113155572224, + "device": 6, + "inactive_split_bytes_current": 11802545152, + "inactive_split_bytes_peak": 13181936128, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:26.766507Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113155572224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113507860992, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113155572224, + "device": 6, + "inactive_split_bytes_current": 10271624192, + "inactive_split_bytes_peak": 13181936128, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:25.569513Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 11802545152, + "inactive_split_bytes_current_delta": 1530920960, + "inactive_split_bytes_peak": 13181936128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 120747720704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120747720704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T142926766976Z.pickle", + "timestamp_utc": "2026-08-01T14:29:26.879154Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T142926766976Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T142926766976Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf4c48715484be8e2d07f1f7d0d5bf394cc0aa27 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T142926766976Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5733ca607ce2634a3f8ad65ee811244f14d17939c48d204f09d1f38df87ba019 +size 6037890 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T142928140335Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T142928140335Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e0fd77f00b911e31fdf4ad4f47b81c49f2057cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T142928140335Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113507860992, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113507860992, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113155572224, + "device": 6, + "inactive_split_bytes_current": 11158719488, + "inactive_split_bytes_peak": 13181936128, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:28.139806Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113155572224, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113507860992, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113155572224, + "device": 6, + "inactive_split_bytes_current": 11802545152, + "inactive_split_bytes_peak": 13181936128, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:26.895235Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 11158719488, + "inactive_split_bytes_current_delta": -643825664, + "inactive_split_bytes_peak": 13181936128, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 120747720704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120747720704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T142928140335Z.pickle", + "timestamp_utc": "2026-08-01T14:29:28.250094Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T142928140335Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T142928140335Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df38fdfa6aabf00146a21774e7d670b7aefbddcb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T142928140335Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bedd3e6de3a50516d3deb9f5262f85c9c8af6964df6cff23a02d4725720d3bf +size 5994675 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T142929737411Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T142929737411Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5593a8f11a0fcfb2c52255d1ffbab7ec999a9bb6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T142929737411Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113507894784, + "active_bytes_peak_delta": 33792, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113507894784, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113155573248, + "device": 6, + "inactive_split_bytes_current": 9980120064, + "inactive_split_bytes_peak": 15562394624, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:29.736951Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113155573248, + "allocated_bytes_peak_delta": 1024, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113507860992, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113155572224, + "device": 6, + "inactive_split_bytes_current": 11158719488, + "inactive_split_bytes_peak": 13181936128, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:28.264768Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9980120064, + "inactive_split_bytes_current_delta": -1178599424, + "inactive_split_bytes_peak": 15562394624, + "inactive_split_bytes_peak_delta": 2380458496, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 120747720704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120747720704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T142929737411Z.pickle", + "timestamp_utc": "2026-08-01T14:29:29.849176Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T142929737411Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T142929737411Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d715ae63b6729568210be3d62c15e50cf20ce030 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T142929737411Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b287318a70806ee335f0f3110ee9285814825b20aded173c428185b05545e91f +size 6036934 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T142931061419Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T142931061419Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9b3948d730794e65c6c0d4d9d8de370dde22b9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T142931061419Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113510908416, + "active_bytes_peak_delta": 3013632, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113510908416, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113158586880, + "device": 6, + "inactive_split_bytes_current": 10038840320, + "inactive_split_bytes_peak": 15562394624, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:31.060978Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113158586880, + "allocated_bytes_peak_delta": 3013632, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113507894784, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113155573248, + "device": 6, + "inactive_split_bytes_current": 9980120064, + "inactive_split_bytes_peak": 15562394624, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:29.864435Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10038840320, + "inactive_split_bytes_current_delta": 58720256, + "inactive_split_bytes_peak": 15562394624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 120747720704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120747720704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T142931061419Z.pickle", + "timestamp_utc": "2026-08-01T14:29:31.166785Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T142931061419Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T142931061419Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c2c02098f0b46ff58077ec26c7ea9ff071f0741 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T142931061419Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e41e06eb9ba9ba0694be286bf1f87d91e1fc5afaba09f6aec0ad7fc473381c7d +size 5994083 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T142932381261Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T142932381261Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fdd38b2d20d0c6bdc7c2fcd1634fffe4c862872c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T142932381261Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113510908416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113510908416, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113158586880, + "device": 6, + "inactive_split_bytes_current": 10451979264, + "inactive_split_bytes_peak": 15562394624, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:32.380747Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113158586880, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113510908416, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113158586880, + "device": 6, + "inactive_split_bytes_current": 10038840320, + "inactive_split_bytes_peak": 15562394624, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:31.180648Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10451979264, + "inactive_split_bytes_current_delta": 413138944, + "inactive_split_bytes_peak": 15562394624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 120747720704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120747720704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T142932381261Z.pickle", + "timestamp_utc": "2026-08-01T14:29:32.499740Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T142932381261Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T142932381261Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e3cd84b478d1c055b2ebb9ae40ab10d048618d8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T142932381261Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4e9a1b81eb7a4c9aa410c1c6b2bca1728499a7c0d59fdee6e174ad8db343a29 +size 6079014 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T142933937848Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T142933937848Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a6c115e97c5da1049b368abb220c0664f5f66815 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T142933937848Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113516807680, + "active_bytes_peak_delta": 5899264, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113516807680, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113164486144, + "device": 6, + "inactive_split_bytes_current": 10921741312, + "inactive_split_bytes_peak": 15562394624, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:33.937375Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113164486144, + "allocated_bytes_peak_delta": 5899264, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113510908416, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113158586880, + "device": 6, + "inactive_split_bytes_current": 10451979264, + "inactive_split_bytes_peak": 15562394624, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:32.517611Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10921741312, + "inactive_split_bytes_current_delta": 469762048, + "inactive_split_bytes_peak": 15562394624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 120747720704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120747720704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T142933937848Z.pickle", + "timestamp_utc": "2026-08-01T14:29:34.052107Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T142933937848Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T142933937848Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7601fb8440c52bf6e6402d4fd25b12a87694f84f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T142933937848Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41071a45a89e42c5d6d94d5f873b5a367fc1fddc315506e8ff453b5a5b92d462 +size 6087606 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T142935269751Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T142935269751Z.json new file mode 100644 index 0000000000000000000000000000000000000000..79efb6b04ced80d711b33ebcf5ade4099347e2f0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T142935269751Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113860608000, + "active_bytes_peak_delta": 343800320, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113860608000, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113164486144, + "device": 6, + "inactive_split_bytes_current": 10451979264, + "inactive_split_bytes_peak": 15562394624, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:35.269266Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113164486144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113516807680, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113164486144, + "device": 6, + "inactive_split_bytes_current": 10921741312, + "inactive_split_bytes_peak": 15562394624, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:34.069329Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10451979264, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 15562394624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 120747720704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120747720704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T142935269751Z.pickle", + "timestamp_utc": "2026-08-01T14:29:35.384405Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T142935269751Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T142935269751Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ccd9c2d58162ad9d43ed7cb73df2d2f95dca56cb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T142935269751Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef9b56d198ceab3ad4924e1167d660b37dc1049044e1b5bde6bce00ff6ff3ad7 +size 6036542 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T143034050824Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T143034050824Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2c5de17df3eec85c5f4e28d542746b5e276b0cd8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T143034050824Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113860608000, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 113860608000, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 113164486144, + "device": 6, + "inactive_split_bytes_current": 11032890368, + "inactive_split_bytes_peak": 15562394624, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:30:34.050268Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113164486144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 113860608000, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 113164486144, + "device": 6, + "inactive_split_bytes_current": 10451979264, + "inactive_split_bytes_peak": 15562394624, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 120747720704, + "reserved_bytes_peak": 120747720704, + "segment_count_current": 486, + "segment_count_peak": 486, + "timestamp_utc": "2026-08-01T14:29:35.401697Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 11032890368, + "inactive_split_bytes_current_delta": 580911104, + "inactive_split_bytes_peak": 15562394624, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 120747720704, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120747720704, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 486, + "segment_count_current_delta": 0, + "segment_count_peak": 486, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T143034050824Z.pickle", + "timestamp_utc": "2026-08-01T14:30:34.281156Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T143034050824Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T143034050824Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c31452daaecfdcb4dfcb1d91cafd64f52ccbc258 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T143034050824Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eb6b78d81ae02dd0e4a98ac0601df297e8b6173824cea300b1be9d5b7089bc8 +size 9741105 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T142923990468Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T142923990468Z.json new file mode 100644 index 0000000000000000000000000000000000000000..49dc7613293098559f1b4e86a07837c89b46f382 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T142923990468Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 37208595968, + "active_bytes_peak": 89874543616, + "active_bytes_peak_delta": 54408207360, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 89874543616, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 89522222080, + "device": 7, + "inactive_split_bytes_current": 8954612224, + "inactive_split_bytes_peak": 12931036160, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 100113842176, + "reserved_bytes_peak": 100113842176, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T14:29:23.989806Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 37208595968, + "allocated_bytes_peak": 89522222080, + "allocated_bytes_peak_delta": 54055885824, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:28:15.713926Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8954612224, + "inactive_split_bytes_current_delta": 8821793280, + "inactive_split_bytes_peak": 12931036160, + "inactive_split_bytes_peak_delta": 12770650112, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 100113842176, + "reserved_bytes_current_delta": 60869836800, + "reserved_bytes_peak": 100113842176, + "reserved_bytes_peak_delta": 60869836800, + "segment_count_current": 420, + "segment_count_current_delta": 349, + "segment_count_peak": 420, + "segment_count_peak_delta": 349, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T142923990468Z.pickle", + "timestamp_utc": "2026-08-01T14:29:24.119125Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T142923990468Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T142923990468Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ca815c8b1a1f9f7faa9c929def0e9d3c6e91b52 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T142923990468Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9d94a90610bf60850c4125627e9fcdddc6e49ccf56e733dd52bda0ade7b0e57 +size 5958327 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T142925433781Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T142925433781Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f8d101ceeef3a02b65b412aa76ee037790a9908 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T142925433781Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113502370304, + "active_bytes_peak_delta": 23627826688, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113502370304, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113150048768, + "device": 7, + "inactive_split_bytes_current": 11244702208, + "inactive_split_bytes_peak": 14541313536, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 120305221632, + "reserved_bytes_peak": 120305221632, + "segment_count_current": 482, + "segment_count_peak": 482, + "timestamp_utc": "2026-08-01T14:29:25.433323Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113150048768, + "allocated_bytes_peak_delta": 23627826688, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 89874543616, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 89522222080, + "device": 7, + "inactive_split_bytes_current": 8954612224, + "inactive_split_bytes_peak": 12931036160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 100113842176, + "reserved_bytes_peak": 100113842176, + "segment_count_current": 420, + "segment_count_peak": 420, + "timestamp_utc": "2026-08-01T14:29:24.143258Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11244702208, + "inactive_split_bytes_current_delta": 2290089984, + "inactive_split_bytes_peak": 14541313536, + "inactive_split_bytes_peak_delta": 1610277376, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 120305221632, + "reserved_bytes_current_delta": 20191379456, + "reserved_bytes_peak": 120305221632, + "reserved_bytes_peak_delta": 20191379456, + "segment_count_current": 482, + "segment_count_current_delta": 62, + "segment_count_peak": 482, + "segment_count_peak_delta": 62, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T142925433781Z.pickle", + "timestamp_utc": "2026-08-01T14:29:25.551279Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T142925433781Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T142925433781Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e01d3e900e453e2a98a9c09c6ab9ff6d110e9af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T142925433781Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28f85c975dde1406c18c3c6390be9cc40881f29714892bc86b6cbe5c400718c7 +size 5950127 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T142926767047Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T142926767047Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f515883f7cf4650fde74b1415ade91bf3b8ccbc9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T142926767047Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113504990720, + "active_bytes_peak_delta": 2620416, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113504990720, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113152669184, + "device": 7, + "inactive_split_bytes_current": 11540400640, + "inactive_split_bytes_peak": 14541313536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 120307318784, + "reserved_bytes_peak": 120307318784, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:26.766508Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113152669184, + "allocated_bytes_peak_delta": 2620416, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113502370304, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113150048768, + "device": 7, + "inactive_split_bytes_current": 11244702208, + "inactive_split_bytes_peak": 14541313536, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 120305221632, + "reserved_bytes_peak": 120305221632, + "segment_count_current": 482, + "segment_count_peak": 482, + "timestamp_utc": "2026-08-01T14:29:25.565850Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11540400640, + "inactive_split_bytes_current_delta": 295698432, + "inactive_split_bytes_peak": 14541313536, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 120307318784, + "reserved_bytes_current_delta": 2097152, + "reserved_bytes_peak": 120307318784, + "reserved_bytes_peak_delta": 2097152, + "segment_count_current": 483, + "segment_count_current_delta": 1, + "segment_count_peak": 483, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T142926767047Z.pickle", + "timestamp_utc": "2026-08-01T14:29:26.890802Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T142926767047Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T142926767047Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..079df64185da2fbaf2f465f69f43b517225ed490 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T142926767047Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3859e141da701df5aefef62807a4648e4f6183fab01b9d4943afb51ccc8d9e1b +size 5951069 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T142928140124Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T142928140124Z.json new file mode 100644 index 0000000000000000000000000000000000000000..325a8fd62b216f7fa2ce5e44ca7787ca3c37dcf5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T142928140124Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113508025344, + "active_bytes_peak_delta": 3034624, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113508025344, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113155703808, + "device": 7, + "inactive_split_bytes_current": 10831563264, + "inactive_split_bytes_peak": 15979430912, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 120307318784, + "reserved_bytes_peak": 120307318784, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:28.139694Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113155703808, + "allocated_bytes_peak_delta": 3034624, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113504990720, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113152669184, + "device": 7, + "inactive_split_bytes_current": 11540400640, + "inactive_split_bytes_peak": 14541313536, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 120307318784, + "reserved_bytes_peak": 120307318784, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:26.916527Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10831563264, + "inactive_split_bytes_current_delta": -708837376, + "inactive_split_bytes_peak": 15979430912, + "inactive_split_bytes_peak_delta": 1438117376, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 120307318784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120307318784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T142928140124Z.pickle", + "timestamp_utc": "2026-08-01T14:29:28.253337Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T142928140124Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T142928140124Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44a9c72ac9b2f6fab6a416818a0b4a5604115e6c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T142928140124Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3755ce124c7b6b9cd3f544513a061ec2c37ede4581ccdb0fe297d1945f1fc01 +size 5953783 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T142929737336Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T142929737336Z.json new file mode 100644 index 0000000000000000000000000000000000000000..00c32fb930f9f210663a4f22c42eb22494bea548 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T142929737336Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113861112832, + "active_bytes_peak_delta": 353087488, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113861112832, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113156469760, + "device": 7, + "inactive_split_bytes_current": 11477486080, + "inactive_split_bytes_peak": 15979430912, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 120307318784, + "reserved_bytes_peak": 120307318784, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:29.736883Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156469760, + "allocated_bytes_peak_delta": 765952, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113508025344, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113155703808, + "device": 7, + "inactive_split_bytes_current": 10831563264, + "inactive_split_bytes_peak": 15979430912, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 120307318784, + "reserved_bytes_peak": 120307318784, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:28.273539Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11477486080, + "inactive_split_bytes_current_delta": 645922816, + "inactive_split_bytes_peak": 15979430912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 120307318784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120307318784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T142929737336Z.pickle", + "timestamp_utc": "2026-08-01T14:29:29.845150Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T142929737336Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T142929737336Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e93e581d7e419c6edd88da9d37e27f75c9153bcd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T142929737336Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:858056bdcfaf7dd1599ce850c4744c7b8ddeaf86b0da5bfbeca7eda77386e236 +size 5953515 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T142931061416Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T142931061416Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c82d3e660d7ecfa21aa7b0b4ba08b1b5dcd330a3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T142931061416Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113861112832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113861112832, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113156469760, + "device": 7, + "inactive_split_bytes_current": 10007382528, + "inactive_split_bytes_peak": 15979430912, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 120307318784, + "reserved_bytes_peak": 120307318784, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:31.060980Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113156469760, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113861112832, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113156469760, + "device": 7, + "inactive_split_bytes_current": 11477486080, + "inactive_split_bytes_peak": 15979430912, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 120307318784, + "reserved_bytes_peak": 120307318784, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:29.858879Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10007382528, + "inactive_split_bytes_current_delta": -1470103552, + "inactive_split_bytes_peak": 15979430912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 120307318784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120307318784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T142931061416Z.pickle", + "timestamp_utc": "2026-08-01T14:29:31.163693Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T142931061416Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T142931061416Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..17ba4886dda16c638af1ca45f2efb9f68c9003f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T142931061416Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96b7850b94fcc901514b88823ff155e6119c2a976a281bb6e53ebc4ba2f84926 +size 5953481 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T142932381333Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T142932381333Z.json new file mode 100644 index 0000000000000000000000000000000000000000..39c4c2fa1ee0639967d42103ecd7e669b6a60fce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T142932381333Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113861112832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113861112832, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113158305280, + "device": 7, + "inactive_split_bytes_current": 11712367104, + "inactive_split_bytes_peak": 15979430912, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 120307318784, + "reserved_bytes_peak": 120307318784, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:32.380903Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113158305280, + "allocated_bytes_peak_delta": 1835520, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113861112832, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113156469760, + "device": 7, + "inactive_split_bytes_current": 10007382528, + "inactive_split_bytes_peak": 15979430912, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 120307318784, + "reserved_bytes_peak": 120307318784, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:31.176383Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11712367104, + "inactive_split_bytes_current_delta": 1704984576, + "inactive_split_bytes_peak": 15979430912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 120307318784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120307318784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T142932381333Z.pickle", + "timestamp_utc": "2026-08-01T14:29:32.480567Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T142932381333Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T142932381333Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..95dd938a9b4d54ce58987aa44388a26f4468278d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T142932381333Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:616f170f72957c8474bba2dad76d472a78d2e06108b4a8d96b3f6048c5ddcba4 +size 5996731 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T142933938027Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T142933938027Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d0419b075a2a84f9ccaab409f1e970280ca67939 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T142933938027Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113861112832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113861112832, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113158870016, + "device": 7, + "inactive_split_bytes_current": 10011576832, + "inactive_split_bytes_peak": 15979430912, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 120307318784, + "reserved_bytes_peak": 120307318784, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:33.937541Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113158870016, + "allocated_bytes_peak_delta": 564736, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113861112832, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113158305280, + "device": 7, + "inactive_split_bytes_current": 11712367104, + "inactive_split_bytes_peak": 15979430912, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 120307318784, + "reserved_bytes_peak": 120307318784, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:32.492394Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10011576832, + "inactive_split_bytes_current_delta": -1700790272, + "inactive_split_bytes_peak": 15979430912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 120307318784, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120307318784, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 483, + "segment_count_current_delta": 0, + "segment_count_peak": 483, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T142933938027Z.pickle", + "timestamp_utc": "2026-08-01T14:29:34.044304Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T142933938027Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T142933938027Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..65e8c81cb06f6b41052e9ae04851713bfd0c0393 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T142933938027Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e68ae6b3bd4703c6892627b1cc33322aa357299d0f6e6e1666429b22219b8a8 +size 6081735 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T142935269988Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T142935269988Z.json new file mode 100644 index 0000000000000000000000000000000000000000..465c24244b21ac29629b6fdd88df1c59a29c6aa5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T142935269988Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 113861112832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113861112832, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113158870016, + "device": 7, + "inactive_split_bytes_current": 10657499648, + "inactive_split_bytes_peak": 15979430912, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 121131499520, + "reserved_bytes_peak": 121131499520, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:35.269507Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 113158870016, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113861112832, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113158870016, + "device": 7, + "inactive_split_bytes_current": 10011576832, + "inactive_split_bytes_peak": 15979430912, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 120307318784, + "reserved_bytes_peak": 120307318784, + "segment_count_current": 483, + "segment_count_peak": 483, + "timestamp_utc": "2026-08-01T14:29:34.056703Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10657499648, + "inactive_split_bytes_current_delta": 645922816, + "inactive_split_bytes_peak": 15979430912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 121131499520, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 121131499520, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 484, + "segment_count_current_delta": 1, + "segment_count_peak": 484, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T142935269988Z.pickle", + "timestamp_utc": "2026-08-01T14:29:35.369648Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T142935269988Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T142935269988Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..73d4d5e7d61432e5c2268b6048702c11a7f2e73d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T142935269988Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:738483ae7ef8bd033724fa1e94dfc4bfe20b37baa713c332cc506ec6d38fd8ea +size 6039126 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T143034050753Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T143034050753Z.json new file mode 100644 index 0000000000000000000000000000000000000000..275b0f583c0464452545e6a7fb79dd08856ec5e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T143034050753Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165762048, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 113861112832, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165762048, + "active_bytes_peak": 113861112832, + "allocated_bytes_current": 61165762048, + "allocated_bytes_peak": 113159222784, + "device": 7, + "inactive_split_bytes_current": 11412474368, + "inactive_split_bytes_peak": 15979430912, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 121131499520, + "reserved_bytes_peak": 121131499520, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:30:34.050309Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165762048, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 113159222784, + "allocated_bytes_peak_delta": 352768, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 113861112832, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 113158870016, + "device": 7, + "inactive_split_bytes_current": 10657499648, + "inactive_split_bytes_peak": 15979430912, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 121131499520, + "reserved_bytes_peak": 121131499520, + "segment_count_current": 484, + "segment_count_peak": 484, + "timestamp_utc": "2026-08-01T14:29:35.381890Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 11412474368, + "inactive_split_bytes_current_delta": 754974720, + "inactive_split_bytes_peak": 15979430912, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 121131499520, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121131499520, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 484, + "segment_count_current_delta": 0, + "segment_count_peak": 484, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T143034050753Z.pickle", + "timestamp_utc": "2026-08-01T14:30:34.263556Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T143034050753Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T143034050753Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..361684966c4a70661366d2e1930c1025bb4bccd7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T143034050753Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:902160b9478dde37388d9a5451864c8a086d395f587b2f198ab5125887e3ae18 +size 9743536 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..52a60b1dcf3a4e1974febba58f01a5122daddbb6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 4096 +capacity: 1.50 +expert_dispatch_overlap: off +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6044 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 225 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T143319442323Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T143319442323Z.json new file mode 100644 index 0000000000000000000000000000000000000000..98b3587d1bcbf229e2c2d7c1738e78ec253dabbe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T143319442323Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854974464, + "active_bytes_current_delta": 31623519232, + "active_bytes_peak": 92108800000, + "active_bytes_peak_delta": 56642463744, + "after": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 92108800000, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 92108800000, + "device": 0, + "inactive_split_bytes_current": 8411810816, + "inactive_split_bytes_peak": 10481519616, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98746499072, + "reserved_bytes_peak": 98746499072, + "segment_count_current": 450, + "segment_count_peak": 450, + "timestamp_utc": "2026-08-01T14:33:19.441901Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854974464, + "allocated_bytes_current_delta": 31623519232, + "allocated_bytes_peak": 92108800000, + "allocated_bytes_peak_delta": 56642463744, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T14:32:11.737563Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8411810816, + "inactive_split_bytes_current_delta": 8276894720, + "inactive_split_bytes_peak": 10481519616, + "inactive_split_bytes_peak_delta": 10320887808, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98746499072, + "reserved_bytes_current_delta": 59500396544, + "reserved_bytes_peak": 98746499072, + "reserved_bytes_peak_delta": 59500396544, + "segment_count_current": 450, + "segment_count_current_delta": 378, + "segment_count_peak": 450, + "segment_count_peak_delta": 378, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T143319442323Z.pickle", + "timestamp_utc": "2026-08-01T14:33:19.553061Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T143319442323Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T143319442323Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e5cc3cb985a2285fcc38e0b667151f1b74cd0097 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T143319442323Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:462461e8f1f69ecaf75b530fc749de81bb0e94c45cb2c0bf0813875562a83c86 +size 5386175 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T143320684118Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T143320684118Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f669f207a2659b33922d487dff91285f8dc6798 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T143320684118Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854974464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115751391232, + "active_bytes_peak_delta": 23642591232, + "after": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115751391232, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115751391232, + "device": 0, + "inactive_split_bytes_current": 8600554496, + "inactive_split_bytes_peak": 10660265984, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:20.683715Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854974464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115751391232, + "allocated_bytes_peak_delta": 23642591232, + "before": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 92108800000, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 92108800000, + "device": 0, + "inactive_split_bytes_current": 8411810816, + "inactive_split_bytes_peak": 10481519616, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98746499072, + "reserved_bytes_peak": 98746499072, + "segment_count_current": 450, + "segment_count_peak": 450, + "timestamp_utc": "2026-08-01T14:33:19.573424Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8600554496, + "inactive_split_bytes_current_delta": 188743680, + "inactive_split_bytes_peak": 10660265984, + "inactive_split_bytes_peak_delta": 178746368, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_current_delta": 19281215488, + "reserved_bytes_peak": 118027714560, + "reserved_bytes_peak_delta": 19281215488, + "segment_count_current": 503, + "segment_count_current_delta": 53, + "segment_count_peak": 503, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T143320684118Z.pickle", + "timestamp_utc": "2026-08-01T14:33:20.798521Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T143320684118Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T143320684118Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2b170ead8fbd76f738985fb07682b90ddb61e3f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T143320684118Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb99a8c627d02371544f9ec303e05a25740be29121eb6808c6dec717fa33c39f +size 5605937 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T143321898514Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T143321898514Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3588e0141f878153c059990eb13de737db227e17 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T143321898514Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854974464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115753611264, + "active_bytes_peak_delta": 2220032, + "after": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115753611264, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115753611264, + "device": 0, + "inactive_split_bytes_current": 8246135808, + "inactive_split_bytes_peak": 10660265984, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:21.898102Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854974464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115753611264, + "allocated_bytes_peak_delta": 2220032, + "before": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115751391232, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115751391232, + "device": 0, + "inactive_split_bytes_current": 8600554496, + "inactive_split_bytes_peak": 10660265984, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:20.814954Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8246135808, + "inactive_split_bytes_current_delta": -354418688, + "inactive_split_bytes_peak": 10660265984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118027714560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T143321898514Z.pickle", + "timestamp_utc": "2026-08-01T14:33:22.003455Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T143321898514Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T143321898514Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f680397322eace5b5e077a819387f175228eff3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T143321898514Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0772a4d1080284079687d51add66fc5cfb251e87743fed0ac5c4ebfb33ffa896 +size 5604655 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T143323121746Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T143323121746Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94553cd4ce5ec27e583c765b14252bc2218a36b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T143323121746Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854974464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115753611264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115753611264, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115753611264, + "device": 0, + "inactive_split_bytes_current": 8537639936, + "inactive_split_bytes_peak": 10724392960, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:23.121330Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854974464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115753611264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115753611264, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115753611264, + "device": 0, + "inactive_split_bytes_current": 8246135808, + "inactive_split_bytes_peak": 10660265984, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:22.018713Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8537639936, + "inactive_split_bytes_current_delta": 291504128, + "inactive_split_bytes_peak": 10724392960, + "inactive_split_bytes_peak_delta": 64126976, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118027714560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T143323121746Z.pickle", + "timestamp_utc": "2026-08-01T14:33:23.225369Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T143323121746Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T143323121746Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b480e9c5de407ef4a414743a621abe0e56e6a384 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T143323121746Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e409a6df926a634cb0db3e67ed0a1bd7bc3891366c047c9d8640a0be04fe6dc +size 5605271 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T143324705443Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T143324705443Z.json new file mode 100644 index 0000000000000000000000000000000000000000..314a45c780dee1040b36ce5803e596a9e06a8bb0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T143324705443Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854974464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115753611264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115753611264, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115753611264, + "device": 0, + "inactive_split_bytes_current": 7832996864, + "inactive_split_bytes_peak": 10724392960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:24.705037Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854974464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115753611264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115753611264, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115753611264, + "device": 0, + "inactive_split_bytes_current": 8537639936, + "inactive_split_bytes_peak": 10724392960, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:23.239985Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7832996864, + "inactive_split_bytes_current_delta": -704643072, + "inactive_split_bytes_peak": 10724392960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118027714560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T143324705443Z.pickle", + "timestamp_utc": "2026-08-01T14:33:24.811989Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T143324705443Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T143324705443Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5f13223e7159ccbb1db4654ad874458d03124696 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T143324705443Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5be473598be4c3f178f86a241105ac2b4b64151a2624477b9ef29ca6c5bf2da2 +size 5603208 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T143325896050Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T143325896050Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27df83f32596c1a0f9955e548c195f90f3ec44c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T143325896050Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854974464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115755859968, + "active_bytes_peak_delta": 2248704, + "after": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115755859968, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115755859968, + "device": 0, + "inactive_split_bytes_current": 7478578176, + "inactive_split_bytes_peak": 10724392960, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:25.895657Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854974464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115755859968, + "allocated_bytes_peak_delta": 2248704, + "before": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115753611264, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115753611264, + "device": 0, + "inactive_split_bytes_current": 7832996864, + "inactive_split_bytes_peak": 10724392960, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:24.826604Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7478578176, + "inactive_split_bytes_current_delta": -354418688, + "inactive_split_bytes_peak": 10724392960, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118027714560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T143325896050Z.pickle", + "timestamp_utc": "2026-08-01T14:33:25.997839Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T143325896050Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T143325896050Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b1a369d73e72c7ec5a961614521d277db8e47fbf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T143325896050Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cfb0231065f364a10b0ccfd67da20e5ffbabb9542a8e8b0bd01ec7bd6b3f277 +size 5604283 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T143327090375Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T143327090375Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e61f677ae8693bd0bbc8598b401ffe42e06f63fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T143327090375Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854974464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115755859968, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115755859968, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115755859968, + "device": 0, + "inactive_split_bytes_current": 9655421952, + "inactive_split_bytes_peak": 11404230656, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:27.089970Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854974464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115755859968, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115755859968, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115755859968, + "device": 0, + "inactive_split_bytes_current": 7478578176, + "inactive_split_bytes_peak": 10724392960, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:26.012340Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9655421952, + "inactive_split_bytes_current_delta": 2176843776, + "inactive_split_bytes_peak": 11404230656, + "inactive_split_bytes_peak_delta": 679837696, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118027714560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T143327090375Z.pickle", + "timestamp_utc": "2026-08-01T14:33:27.192192Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T143327090375Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T143327090375Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..20fecbedb25eb7c2c33686a7b311ffc2d0bd5363 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T143327090375Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef693822dc33cd9384b1f67eb47159d32e58cd0cdaa62636aafd603c63f3c09c +size 5604682 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T143328276435Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T143328276435Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6a6b5b7a5068a81c4a5be07375b62986424aa462 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T143328276435Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854974464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115755859968, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115755859968, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115755859968, + "device": 0, + "inactive_split_bytes_current": 9005304832, + "inactive_split_bytes_peak": 11404230656, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:28.276032Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854974464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115755859968, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115755859968, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115755859968, + "device": 0, + "inactive_split_bytes_current": 9655421952, + "inactive_split_bytes_peak": 11404230656, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:27.206705Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9005304832, + "inactive_split_bytes_current_delta": -650117120, + "inactive_split_bytes_peak": 11404230656, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118027714560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T143328276435Z.pickle", + "timestamp_utc": "2026-08-01T14:33:28.388593Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T143328276435Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T143328276435Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c2693a60ac44649fe46a4904ab224d3ed3934186 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T143328276435Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d52dfca74815480a5d2290e9a9d7d38b417168b629cb925ee3f6ed1fd2cfe20 +size 5607549 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T143329471127Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T143329471127Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8bbcc46743c2636e85f7daa8721c320a3d5a8a51 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T143329471127Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854974464, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115757543424, + "active_bytes_peak_delta": 1683456, + "after": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115757543424, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115757543424, + "device": 0, + "inactive_split_bytes_current": 8306953216, + "inactive_split_bytes_peak": 11404230656, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:29.470709Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854974464, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115757543424, + "allocated_bytes_peak_delta": 1683456, + "before": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115755859968, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115755859968, + "device": 0, + "inactive_split_bytes_current": 9005304832, + "inactive_split_bytes_peak": 11404230656, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:28.407772Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8306953216, + "inactive_split_bytes_current_delta": -698351616, + "inactive_split_bytes_peak": 11404230656, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118027714560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T143329471127Z.pickle", + "timestamp_utc": "2026-08-01T14:33:29.575714Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T143329471127Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T143329471127Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb947a4b7408c3871d697450e4966bcee858988b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T143329471127Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:726afc7ae26ff5b86458557ee384ea191e2f367683395df1f717f2ba5b9040b8 +size 5603530 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T143425847933Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T143425847933Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d59d1d1fe437d017f1379fe1352a64a4ffaa3be4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T143425847933Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804331520, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115759771648, + "active_bytes_peak_delta": 2228224, + "after": { + "active_bytes_current": 66804331520, + "active_bytes_peak": 115759771648, + "allocated_bytes_current": 66804331520, + "allocated_bytes_peak": 115759771648, + "device": 0, + "inactive_split_bytes_current": 7111887872, + "inactive_split_bytes_peak": 11404230656, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:34:25.847460Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804331520, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115759771648, + "allocated_bytes_peak_delta": 2228224, + "before": { + "active_bytes_current": 66854974464, + "active_bytes_peak": 115757543424, + "allocated_bytes_current": 66854974464, + "allocated_bytes_peak": 115757543424, + "device": 0, + "inactive_split_bytes_current": 8306953216, + "inactive_split_bytes_peak": 11404230656, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_peak": 118027714560, + "segment_count_current": 503, + "segment_count_peak": 503, + "timestamp_utc": "2026-08-01T14:33:29.591733Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7111887872, + "inactive_split_bytes_current_delta": -1195065344, + "inactive_split_bytes_peak": 11404230656, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118027714560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118027714560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 503, + "segment_count_current_delta": 0, + "segment_count_peak": 503, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T143425847933Z.pickle", + "timestamp_utc": "2026-08-01T14:34:26.045604Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T143425847933Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T143425847933Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..97fae7032641edcdfe4798f15c70fc3e6b9c3c26 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T143425847933Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8aa1b66ad4cb7c58e4ea5e3cab91adc566ae7f68f866c8b01938dd08e3ddac09 +size 9801524 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T143319442357Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T143319442357Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d11c5468d0c9bbc6588e8d1df9ab479e20e060ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T143319442357Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 92103927296, + "active_bytes_peak_delta": 56637591040, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92103927296, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92103927296, + "device": 1, + "inactive_split_bytes_current": 7528238080, + "inactive_split_bytes_peak": 11416151040, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 98333360128, + "reserved_bytes_peak": 98333360128, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T14:33:19.441775Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 92103927296, + "allocated_bytes_peak_delta": 56637591040, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:32:11.735801Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7528238080, + "inactive_split_bytes_current_delta": 7395419136, + "inactive_split_bytes_peak": 11416151040, + "inactive_split_bytes_peak_delta": 11255764992, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 98333360128, + "reserved_bytes_current_delta": 59089354752, + "reserved_bytes_peak": 98333360128, + "reserved_bytes_peak_delta": 59089354752, + "segment_count_current": 448, + "segment_count_current_delta": 377, + "segment_count_peak": 448, + "segment_count_peak_delta": 377, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T143319442357Z.pickle", + "timestamp_utc": "2026-08-01T14:33:19.556705Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T143319442357Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T143319442357Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6286406e62cda93466f9bc844a338b0cd9dfd73a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T143319442357Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcb4685a0f406cc686d7ba19593c129db8470f8bca658665d7d5f0003a39a2d7 +size 5374507 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T143320684405Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T143320684405Z.json new file mode 100644 index 0000000000000000000000000000000000000000..309fb558ff4bf753080fc917da2e899b6b5bd781 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T143320684405Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115750526976, + "active_bytes_peak_delta": 23646599680, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115750526976, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115750526976, + "device": 1, + "inactive_split_bytes_current": 7826033664, + "inactive_split_bytes_peak": 11598958592, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:20.683871Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115750526976, + "allocated_bytes_peak_delta": 23646599680, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92103927296, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92103927296, + "device": 1, + "inactive_split_bytes_current": 7528238080, + "inactive_split_bytes_peak": 11416151040, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 98333360128, + "reserved_bytes_peak": 98333360128, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T14:33:19.576605Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7826033664, + "inactive_split_bytes_current_delta": 297795584, + "inactive_split_bytes_peak": 11598958592, + "inactive_split_bytes_peak_delta": 182807552, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 19746783232, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 19746783232, + "segment_count_current": 500, + "segment_count_current_delta": 52, + "segment_count_peak": 500, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T143320684405Z.pickle", + "timestamp_utc": "2026-08-01T14:33:20.800744Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T143320684405Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T143320684405Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a7dcd863f1816796110e46cd32111c522ff2beda --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T143320684405Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5179e372e2ab3e826ac70a03bc767ee6f6310a10cc6c110d8837c2d71d9d27cd +size 5598694 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T143321898730Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T143321898730Z.json new file mode 100644 index 0000000000000000000000000000000000000000..35c2bb496c637568099ea9d04569d25f942b0762 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T143321898730Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115755810816, + "active_bytes_peak_delta": 5283840, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115755810816, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115755810816, + "device": 1, + "inactive_split_bytes_current": 8824278016, + "inactive_split_bytes_peak": 11598958592, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:21.898315Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115755810816, + "allocated_bytes_peak_delta": 5283840, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115750526976, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115750526976, + "device": 1, + "inactive_split_bytes_current": 7826033664, + "inactive_split_bytes_peak": 11598958592, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:20.816322Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8824278016, + "inactive_split_bytes_current_delta": 998244352, + "inactive_split_bytes_peak": 11598958592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T143321898730Z.pickle", + "timestamp_utc": "2026-08-01T14:33:22.007471Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T143321898730Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T143321898730Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5e2c277ffab8d6f8dff56762a9e79daaea9f0bfe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T143321898730Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daeb0287a41017a153766903ec7eeb32922abcb8312146210ea4af5922b1071a +size 5600764 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T143323121905Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T143323121905Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1f6c18fba582a9c25038a6eefb263efe49c2efe0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T143323121905Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115758391296, + "active_bytes_peak_delta": 2580480, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115758391296, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115758391296, + "device": 1, + "inactive_split_bytes_current": 10298575872, + "inactive_split_bytes_peak": 11598958592, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:23.121383Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115758391296, + "allocated_bytes_peak_delta": 2580480, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115755810816, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115755810816, + "device": 1, + "inactive_split_bytes_current": 8824278016, + "inactive_split_bytes_peak": 11598958592, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:22.022590Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10298575872, + "inactive_split_bytes_current_delta": 1474297856, + "inactive_split_bytes_peak": 11598958592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T143323121905Z.pickle", + "timestamp_utc": "2026-08-01T14:33:23.252263Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T143323121905Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T143323121905Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ebeba96be0a056159956594aff522383968438a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T143323121905Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0634ebd5efc319bc0296c5e877b63cc87dcb2633140f5ea516b63c3769946adb +size 5597944 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T143324705565Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T143324705565Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8bbc5c4b363d36874958f36b61388d3e2242c4fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T143324705565Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759439872, + "active_bytes_peak_delta": 1048576, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759439872, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759439872, + "device": 1, + "inactive_split_bytes_current": 8765557760, + "inactive_split_bytes_peak": 11598958592, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:24.705074Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759439872, + "allocated_bytes_peak_delta": 1048576, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115758391296, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115758391296, + "device": 1, + "inactive_split_bytes_current": 10298575872, + "inactive_split_bytes_peak": 11598958592, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:23.275770Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8765557760, + "inactive_split_bytes_current_delta": -1533018112, + "inactive_split_bytes_peak": 11598958592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T143324705565Z.pickle", + "timestamp_utc": "2026-08-01T14:33:24.812996Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T143324705565Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T143324705565Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b9c5782e0188f87740bda2bf9a128e8cad15dc00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T143324705565Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2eecae504b138e317c8f6b83c646179e1f6acc8181843cf42bc5861c7717059 +size 5570361 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T143325896559Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T143325896559Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8db24de711b1af656b74cbda7222f78fefa5b113 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T143325896559Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759439872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759439872, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759439872, + "device": 1, + "inactive_split_bytes_current": 8006388736, + "inactive_split_bytes_peak": 11598958592, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:25.895824Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759439872, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759439872, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759439872, + "device": 1, + "inactive_split_bytes_current": 8765557760, + "inactive_split_bytes_peak": 11598958592, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:24.826460Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8006388736, + "inactive_split_bytes_current_delta": -759169024, + "inactive_split_bytes_peak": 11598958592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T143325896559Z.pickle", + "timestamp_utc": "2026-08-01T14:33:26.001344Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T143325896559Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T143325896559Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ca0a8053b3cffd5a662cec926aea3335a70264e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T143325896559Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27a6f8ac3746ad945f0d4c1b37b64b6e803579d54e2bb9efd328d84dd19c23c8 +size 5581545 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T143327090560Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T143327090560Z.json new file mode 100644 index 0000000000000000000000000000000000000000..42bf63bae05f66868bddc5549d293fdb6ac470ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T143327090560Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759439872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759439872, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759439872, + "device": 1, + "inactive_split_bytes_current": 9887534080, + "inactive_split_bytes_peak": 11598958592, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:27.090007Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759439872, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759439872, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759439872, + "device": 1, + "inactive_split_bytes_current": 8006388736, + "inactive_split_bytes_peak": 11598958592, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:26.014973Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9887534080, + "inactive_split_bytes_current_delta": 1881145344, + "inactive_split_bytes_peak": 11598958592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T143327090560Z.pickle", + "timestamp_utc": "2026-08-01T14:33:27.194932Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T143327090560Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T143327090560Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bf2e20fd3873f96a733f9d3dd21b3594cfcc1486 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T143327090560Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3a2f1e96311215cb9048d2074d786e85c53bdcca383add9b2da0370f1ba84b7 +size 5578731 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T143328276528Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T143328276528Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9319f84f50e589e1dc2eb87ec8f698e9a3852257 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T143328276528Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759439872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759439872, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759439872, + "device": 1, + "inactive_split_bytes_current": 7888948224, + "inactive_split_bytes_peak": 11598958592, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:28.276035Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759439872, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759439872, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759439872, + "device": 1, + "inactive_split_bytes_current": 9887534080, + "inactive_split_bytes_peak": 11598958592, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:27.208788Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7888948224, + "inactive_split_bytes_current_delta": -1998585856, + "inactive_split_bytes_peak": 11598958592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T143328276528Z.pickle", + "timestamp_utc": "2026-08-01T14:33:28.378512Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T143328276528Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T143328276528Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5c4c33970fdb906c9d1acce2d2e7f121491cb5ea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T143328276528Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03840e0e975d2dac09ab18dabc3a559797e44dfcc4a99ff8667a1ea2a8a239d1 +size 5570675 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T143329471306Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T143329471306Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70dd359e377c3f1681b1dbd9eb904ed2efe13379 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T143329471306Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759439872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759439872, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759439872, + "device": 1, + "inactive_split_bytes_current": 7003950080, + "inactive_split_bytes_peak": 11598958592, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:29.470744Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759439872, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759439872, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759439872, + "device": 1, + "inactive_split_bytes_current": 7888948224, + "inactive_split_bytes_peak": 11598958592, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:28.391321Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7003950080, + "inactive_split_bytes_current_delta": -884998144, + "inactive_split_bytes_peak": 11598958592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T143329471306Z.pickle", + "timestamp_utc": "2026-08-01T14:33:29.575977Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T143329471306Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T143329471306Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..392761a60925b45654a11a88f84bf40a917eb28b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T143329471306Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3289d9413eb6f805f7b72cae169773e56e35440f2193365743f53b8e38b7113 +size 5584715 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T143425847977Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T143425847977Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0ceaca6fb3fd968d162080d0afd69b19b016f70b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T143425847977Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115759439872, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 115759439872, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 115759439872, + "device": 1, + "inactive_split_bytes_current": 6234606592, + "inactive_split_bytes_peak": 11598958592, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:34:25.847495Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115759439872, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759439872, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759439872, + "device": 1, + "inactive_split_bytes_current": 7003950080, + "inactive_split_bytes_peak": 11598958592, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:29.589655Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6234606592, + "inactive_split_bytes_current_delta": -769343488, + "inactive_split_bytes_peak": 11598958592, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T143425847977Z.pickle", + "timestamp_utc": "2026-08-01T14:34:26.058254Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T143425847977Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T143425847977Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..baa189cf14597e5b148ab018a47e4ddaddb0bd1a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T143425847977Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3830b54b286ffc0ab2683b5bb793c029014172c7f8d653402348bbc72f9147bc +size 9186461 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T143319442279Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T143319442279Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ccb29190800f11dd9e0a8999ad1c9d95268f3d04 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T143319442279Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 92103927296, + "active_bytes_peak_delta": 56637591040, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92103927296, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92103927296, + "device": 2, + "inactive_split_bytes_current": 7524043776, + "inactive_split_bytes_peak": 11416077312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98333360128, + "reserved_bytes_peak": 98333360128, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T14:33:19.441654Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 92103927296, + "allocated_bytes_peak_delta": 56637591040, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:32:11.735756Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7524043776, + "inactive_split_bytes_current_delta": 7391224832, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 11255691264, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98333360128, + "reserved_bytes_current_delta": 59089354752, + "reserved_bytes_peak": 98333360128, + "reserved_bytes_peak_delta": 59089354752, + "segment_count_current": 448, + "segment_count_current_delta": 377, + "segment_count_peak": 448, + "segment_count_peak_delta": 377, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T143319442279Z.pickle", + "timestamp_utc": "2026-08-01T14:33:19.553621Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T143319442279Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T143319442279Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e27c953d48d262f2b5acbe021c95824d77ae4ba3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T143319442279Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bbbd6497859a79537be9123bd8b22d8125d0c2c42ee5aef280cb7141bde9385 +size 5377238 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T143320684079Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T143320684079Z.json new file mode 100644 index 0000000000000000000000000000000000000000..15ac41a6b01a5d196ee916db848d35d6ff27b44c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T143320684079Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115756294144, + "active_bytes_peak_delta": 23652366848, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115756294144, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115756294144, + "device": 2, + "inactive_split_bytes_current": 8648117248, + "inactive_split_bytes_peak": 11416077312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:20.683673Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115756294144, + "allocated_bytes_peak_delta": 23652366848, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92103927296, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92103927296, + "device": 2, + "inactive_split_bytes_current": 7524043776, + "inactive_split_bytes_peak": 11416077312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98333360128, + "reserved_bytes_peak": 98333360128, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T14:33:19.573088Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8648117248, + "inactive_split_bytes_current_delta": 1124073472, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 19805503488, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 19805503488, + "segment_count_current": 501, + "segment_count_current_delta": 53, + "segment_count_peak": 501, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T143320684079Z.pickle", + "timestamp_utc": "2026-08-01T14:33:20.794502Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T143320684079Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T143320684079Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..02a8f6d15a412f2b83fd0cc666f74b2bbf075a5b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T143320684079Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20529e59762d4dcb4bf717bbe452a56c798bba0661b25a2ea425ca276a9f4206 +size 5567612 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T143321898601Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T143321898601Z.json new file mode 100644 index 0000000000000000000000000000000000000000..03c7ffa9a71f483d47c62c279fa243bc3c9a928d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T143321898601Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115756294144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115756294144, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115756294144, + "device": 2, + "inactive_split_bytes_current": 9111587840, + "inactive_split_bytes_peak": 11416077312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:21.898149Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115756294144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115756294144, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115756294144, + "device": 2, + "inactive_split_bytes_current": 8648117248, + "inactive_split_bytes_peak": 11416077312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:20.808340Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9111587840, + "inactive_split_bytes_current_delta": 463470592, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T143321898601Z.pickle", + "timestamp_utc": "2026-08-01T14:33:22.001037Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T143321898601Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T143321898601Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d522f9bd0c962dcc35fb95ba40355e4689668c15 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T143321898601Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2071df3626b5c9761ae62f589c1f3a113394a221f5092c3fd0187f439638c649 +size 5601727 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T143323121710Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T143323121710Z.json new file mode 100644 index 0000000000000000000000000000000000000000..47560946d6bd3a3882e821ec06c38c58125fbd22 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T143323121710Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115756294144, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115756294144, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115756294144, + "device": 2, + "inactive_split_bytes_current": 8232881152, + "inactive_split_bytes_peak": 11416077312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:23.121308Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115756294144, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115756294144, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115756294144, + "device": 2, + "inactive_split_bytes_current": 9111587840, + "inactive_split_bytes_peak": 11416077312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:22.015170Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8232881152, + "inactive_split_bytes_current_delta": -878706688, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T143323121710Z.pickle", + "timestamp_utc": "2026-08-01T14:33:23.224262Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T143323121710Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T143323121710Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..253a4013d9a9e8a6e6bfd0320202d8d2a83db38d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T143323121710Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a79af64169a5f426202dcc11d6ab0276d9892447224a99b8c12a7e9d01d465ad +size 5597180 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T143324705434Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T143324705434Z.json new file mode 100644 index 0000000000000000000000000000000000000000..90d94c27abf8df6299f7a990680e71b3fa5340de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T143324705434Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760992256, + "active_bytes_peak_delta": 4698112, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115760992256, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115760992256, + "device": 2, + "inactive_split_bytes_current": 7893142528, + "inactive_split_bytes_peak": 11416077312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:24.705024Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760992256, + "allocated_bytes_peak_delta": 4698112, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115756294144, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115756294144, + "device": 2, + "inactive_split_bytes_current": 8232881152, + "inactive_split_bytes_peak": 11416077312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:23.237200Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7893142528, + "inactive_split_bytes_current_delta": -339738624, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T143324705434Z.pickle", + "timestamp_utc": "2026-08-01T14:33:24.810581Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T143324705434Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T143324705434Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df4f863ed17ab63927cd9a6aa8b029e61365aca0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T143324705434Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69d97800d887d3e72ddcdb3bd0d946f57ab72e75dc08345e47ed18bddba8dcab +size 5600063 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T143325896038Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T143325896038Z.json new file mode 100644 index 0000000000000000000000000000000000000000..052dcf149ae063765f07eef05800d28d2bc704b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T143325896038Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760992256, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115760992256, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115760992256, + "device": 2, + "inactive_split_bytes_current": 9067547648, + "inactive_split_bytes_peak": 11618634752, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:25.895636Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760992256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115760992256, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115760992256, + "device": 2, + "inactive_split_bytes_current": 7893142528, + "inactive_split_bytes_peak": 11416077312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:24.823636Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9067547648, + "inactive_split_bytes_current_delta": 1174405120, + "inactive_split_bytes_peak": 11618634752, + "inactive_split_bytes_peak_delta": 202557440, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T143325896038Z.pickle", + "timestamp_utc": "2026-08-01T14:33:25.994401Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T143325896038Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T143325896038Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44350ede06d264b8e2fa15a6d1fb18e9e37bf001 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T143325896038Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cfbfcec68accd728c826178eb743233d41f55b446185c8ca58b6507857a868c +size 5601136 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T143327090365Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T143327090365Z.json new file mode 100644 index 0000000000000000000000000000000000000000..61eaa681efa72d35e67ca64e55b6826d6be87f24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T143327090365Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760992256, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115760992256, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115760992256, + "device": 2, + "inactive_split_bytes_current": 10174843904, + "inactive_split_bytes_peak": 11618634752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:27.089956Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760992256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115760992256, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115760992256, + "device": 2, + "inactive_split_bytes_current": 9067547648, + "inactive_split_bytes_peak": 11618634752, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:26.007047Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10174843904, + "inactive_split_bytes_current_delta": 1107296256, + "inactive_split_bytes_peak": 11618634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T143327090365Z.pickle", + "timestamp_utc": "2026-08-01T14:33:27.188304Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T143327090365Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T143327090365Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b64051c331f71e2baf1230e97728c600c6a60e7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T143327090365Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f46c205abe0a3e69aaa3a598db0f2b09f4b4b2080aee56028396009f2b6674a1 +size 5600221 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T143328276414Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T143328276414Z.json new file mode 100644 index 0000000000000000000000000000000000000000..47aa1060775040b880c5c8e2dc0a507f51831246 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T143328276414Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760992256, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115760992256, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115760992256, + "device": 2, + "inactive_split_bytes_current": 8885095424, + "inactive_split_bytes_peak": 11618634752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:28.276010Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760992256, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115760992256, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115760992256, + "device": 2, + "inactive_split_bytes_current": 10174843904, + "inactive_split_bytes_peak": 11618634752, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:27.200963Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8885095424, + "inactive_split_bytes_current_delta": -1289748480, + "inactive_split_bytes_peak": 11618634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T143328276414Z.pickle", + "timestamp_utc": "2026-08-01T14:33:28.375360Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T143328276414Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T143328276414Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6be553aef7596ad39e9b33e505f0028efdd30528 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T143328276414Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c4b3ffbf70a6a26e13bfcf731bd47fbc9981d8e2eeb831716f928bc3286bd62 +size 5600119 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T143329471322Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T143329471322Z.json new file mode 100644 index 0000000000000000000000000000000000000000..94a16163c2cb9713f2d332d5def3378157794803 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T143329471322Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115761760768, + "active_bytes_peak_delta": 768512, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115761760768, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115761760768, + "device": 2, + "inactive_split_bytes_current": 9000438784, + "inactive_split_bytes_peak": 11618634752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:29.470749Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115761760768, + "allocated_bytes_peak_delta": 768512, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115760992256, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115760992256, + "device": 2, + "inactive_split_bytes_current": 8885095424, + "inactive_split_bytes_peak": 11618634752, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:28.390431Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9000438784, + "inactive_split_bytes_current_delta": 115343360, + "inactive_split_bytes_peak": 11618634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T143329471322Z.pickle", + "timestamp_utc": "2026-08-01T14:33:29.572535Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T143329471322Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T143329471322Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..61b9b630dcf26e37917653cc1b940a01b4467d48 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T143329471322Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f58cc981213f524ed81dd58b2fce1f1d82dd74b62da05b8cf6fcb6f925448cb0 +size 5600108 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T143425847931Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T143425847931Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f97a6dda86e365bb66545a3c35d49b7a771ec6db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T143425847931Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115761760768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 115761760768, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 115761760768, + "device": 2, + "inactive_split_bytes_current": 6234606592, + "inactive_split_bytes_peak": 11618634752, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:34:25.847468Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115761760768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115761760768, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115761760768, + "device": 2, + "inactive_split_bytes_current": 9000438784, + "inactive_split_bytes_peak": 11618634752, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:29.585435Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6234606592, + "inactive_split_bytes_current_delta": -2765832192, + "inactive_split_bytes_peak": 11618634752, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T143425847931Z.pickle", + "timestamp_utc": "2026-08-01T14:34:26.049962Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T143425847931Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T143425847931Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4ecb3b4b36f3786b29bd379ea1921ef4b19b933f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T143425847931Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d91a9d3cac59e27a2dadad7c58588b4c438b758eec3d8541bd1582aad71ebb1 +size 9755955 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T143319442300Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T143319442300Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2f6140ab1823959d79edca13abccca527c090b74 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T143319442300Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 92103927296, + "active_bytes_peak_delta": 56637591040, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92103927296, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92103927296, + "device": 3, + "inactive_split_bytes_current": 7524043776, + "inactive_split_bytes_peak": 11416077312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 98333360128, + "reserved_bytes_peak": 98333360128, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T14:33:19.441665Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 92103927296, + "allocated_bytes_peak_delta": 56637591040, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:32:11.736211Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7524043776, + "inactive_split_bytes_current_delta": 7391224832, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 11255691264, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 98333360128, + "reserved_bytes_current_delta": 59089354752, + "reserved_bytes_peak": 98333360128, + "reserved_bytes_peak_delta": 59089354752, + "segment_count_current": 448, + "segment_count_current_delta": 377, + "segment_count_peak": 448, + "segment_count_peak_delta": 377, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T143319442300Z.pickle", + "timestamp_utc": "2026-08-01T14:33:19.549389Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T143319442300Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T143319442300Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..044a93f66904ca3825d42b54c1666b0a553858ce --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T143319442300Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a56e54f4105da5db10e77ee696c204351cd2df96b41d86441cb47e1f163ab0c +size 5377235 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T143320684130Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T143320684130Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ec728b3bafcaf69dcad434a31f41797a782b20ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T143320684130Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115750526976, + "active_bytes_peak_delta": 23646599680, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115750526976, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115750526976, + "device": 3, + "inactive_split_bytes_current": 7945571328, + "inactive_split_bytes_peak": 11596861440, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:20.683674Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115750526976, + "allocated_bytes_peak_delta": 23646599680, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92103927296, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92103927296, + "device": 3, + "inactive_split_bytes_current": 7524043776, + "inactive_split_bytes_peak": 11416077312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 98333360128, + "reserved_bytes_peak": 98333360128, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T14:33:19.565607Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7945571328, + "inactive_split_bytes_current_delta": 421527552, + "inactive_split_bytes_peak": 11596861440, + "inactive_split_bytes_peak_delta": 180784128, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 19746783232, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 19746783232, + "segment_count_current": 500, + "segment_count_current_delta": 52, + "segment_count_peak": 500, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T143320684130Z.pickle", + "timestamp_utc": "2026-08-01T14:33:20.794597Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T143320684130Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T143320684130Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ad85a17276e9bdf6acfdb2b5d4a878a7fd10b76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T143320684130Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51822068dac6a9a4eaa0ba8f61f1877c13b26c889419561411475dd7599837f7 +size 5602085 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T143321898485Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T143321898485Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7696b5f3d6b769e2f333378c39be57377dff746a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T143321898485Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115755810816, + "active_bytes_peak_delta": 5283840, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115755810816, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115755810816, + "device": 3, + "inactive_split_bytes_current": 9124170752, + "inactive_split_bytes_peak": 11596861440, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:21.898082Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115755810816, + "allocated_bytes_peak_delta": 5283840, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115750526976, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115750526976, + "device": 3, + "inactive_split_bytes_current": 7945571328, + "inactive_split_bytes_peak": 11596861440, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:20.807306Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9124170752, + "inactive_split_bytes_current_delta": 1178599424, + "inactive_split_bytes_peak": 11596861440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T143321898485Z.pickle", + "timestamp_utc": "2026-08-01T14:33:21.996314Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T143321898485Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T143321898485Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9af781754ca6f7489479fae61edf4361a94dd4ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T143321898485Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac39d2900bea8031abbea9b74fb35329bfef6671a1fdb7a6efef170e74a38f26 +size 5583294 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T143323121725Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T143323121725Z.json new file mode 100644 index 0000000000000000000000000000000000000000..60cac93cfe5ae278e73d9420f6fc0832a651494c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T143323121725Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115757867008, + "active_bytes_peak_delta": 2056192, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115757867008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115757867008, + "device": 3, + "inactive_split_bytes_current": 9119976448, + "inactive_split_bytes_peak": 11596861440, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:23.121301Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115757867008, + "allocated_bytes_peak_delta": 2056192, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115755810816, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115755810816, + "device": 3, + "inactive_split_bytes_current": 9124170752, + "inactive_split_bytes_peak": 11596861440, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:22.007801Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9119976448, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 11596861440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T143323121725Z.pickle", + "timestamp_utc": "2026-08-01T14:33:23.218646Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T143323121725Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T143323121725Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9f4a704a0cfecc0d1085dbf00334dbd0ee730bb7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T143323121725Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9bc12e23564bad0a1f6cb3979bbe8921bb2075532969456fdcafc00982574d2a +size 5600105 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T143324705457Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T143324705457Z.json new file mode 100644 index 0000000000000000000000000000000000000000..870b914adec4f237b107960beaa3484812992e18 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T143324705457Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759722496, + "active_bytes_peak_delta": 1855488, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 3, + "inactive_split_bytes_current": 8297892864, + "inactive_split_bytes_peak": 11596861440, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:24.705028Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759722496, + "allocated_bytes_peak_delta": 1855488, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115757867008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115757867008, + "device": 3, + "inactive_split_bytes_current": 9119976448, + "inactive_split_bytes_peak": 11596861440, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:23.231190Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8297892864, + "inactive_split_bytes_current_delta": -822083584, + "inactive_split_bytes_peak": 11596861440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T143324705457Z.pickle", + "timestamp_utc": "2026-08-01T14:33:24.809599Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T143324705457Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T143324705457Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6b5336ec89ca9328cf3f975fd67907c446df0832 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T143324705457Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6a93facce8f22e09cf336975951a3872db0abafc91dda4ba92b00ee7adff7f5 +size 5600297 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T143325900170Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T143325900170Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dfd67d34b1bd681bd7800c2a3e792fef2af3e477 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T143325900170Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 3, + "inactive_split_bytes_current": 9050770432, + "inactive_split_bytes_peak": 11596861440, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:25.899739Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 3, + "inactive_split_bytes_current": 8297892864, + "inactive_split_bytes_peak": 11596861440, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:24.822644Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9050770432, + "inactive_split_bytes_current_delta": 752877568, + "inactive_split_bytes_peak": 11596861440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T143325900170Z.pickle", + "timestamp_utc": "2026-08-01T14:33:26.003958Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T143325900170Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T143325900170Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d756519a4b0d7aee042209bda756d069ce86de76 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T143325900170Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68ffac1ab6106cd0af24f0634ad79dbbc105f1b2859408f3c4bd4a19f5c95799 +size 5599875 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T143327090362Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T143327090362Z.json new file mode 100644 index 0000000000000000000000000000000000000000..70c91141bda0917004a62ff221d4ec8e08c6b169 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T143327090362Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 3, + "inactive_split_bytes_current": 7538723840, + "inactive_split_bytes_peak": 11596861440, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:27.089945Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 3, + "inactive_split_bytes_current": 9050770432, + "inactive_split_bytes_peak": 11596861440, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:26.019574Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7538723840, + "inactive_split_bytes_current_delta": -1512046592, + "inactive_split_bytes_peak": 11596861440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T143327090362Z.pickle", + "timestamp_utc": "2026-08-01T14:33:27.190040Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T143327090362Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T143327090362Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..677179b8f484ef89fa1dcc250c87fd7afc15ec49 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T143327090362Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eccf16d1ba564f30bcbea9cee75a0f477f48df5d1bd6380203be1df14e3f346 +size 5600350 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T143328276421Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T143328276421Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8a881f43dbb43499932906c4652b7d9f53072904 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T143328276421Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 3, + "inactive_split_bytes_current": 9002535936, + "inactive_split_bytes_peak": 11596861440, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:28.276009Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 3, + "inactive_split_bytes_current": 7538723840, + "inactive_split_bytes_peak": 11596861440, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:27.202571Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9002535936, + "inactive_split_bytes_current_delta": 1463812096, + "inactive_split_bytes_peak": 11596861440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T143328276421Z.pickle", + "timestamp_utc": "2026-08-01T14:33:28.374465Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T143328276421Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T143328276421Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5fb90e9ab03b6176e7c1c063a1869b21908d11f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T143328276421Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcf564e1e8dddf71492d76120bebb4226613210d3472da7cb110250578818e28 +size 5568665 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T143329471127Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T143329471127Z.json new file mode 100644 index 0000000000000000000000000000000000000000..048dbcd27791dd38dea54f0a0492dd141a85087e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T143329471127Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 3, + "inactive_split_bytes_current": 9476492288, + "inactive_split_bytes_peak": 11596861440, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:29.470695Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 3, + "inactive_split_bytes_current": 9002535936, + "inactive_split_bytes_peak": 11596861440, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:28.386833Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9476492288, + "inactive_split_bytes_current_delta": 473956352, + "inactive_split_bytes_peak": 11596861440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T143329471127Z.pickle", + "timestamp_utc": "2026-08-01T14:33:29.574134Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T143329471127Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T143329471127Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5634c7c54466e7f1e6b0d30f7995289bac44b547 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T143329471127Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6646d54f9476ab3c2f2f9c7708f23fc0247b6f37c54862b0b45a033cc8482f12 +size 5596862 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T143425848022Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T143425848022Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6e16a532385d441d35c50ed55e8e60e469548127 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T143425848022Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115759722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 115759722496, + "device": 3, + "inactive_split_bytes_current": 5999725568, + "inactive_split_bytes_peak": 11596861440, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:34:25.847461Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115759722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 3, + "inactive_split_bytes_current": 9476492288, + "inactive_split_bytes_peak": 11596861440, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:29.586971Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5999725568, + "inactive_split_bytes_current_delta": -3476766720, + "inactive_split_bytes_peak": 11596861440, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T143425848022Z.pickle", + "timestamp_utc": "2026-08-01T14:34:26.047699Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T143425848022Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T143425848022Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..45cad794d3677b2fa1d58fe9639f115cd21abd2b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T143425848022Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03561a0666ca8cc2772f49a3da0bd69d5d79f7c88fe433dbeffb27132b5ea1bf +size 9171933 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T143319442153Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T143319442153Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69eb9795af4d7a7ad7773a7605729069539f3d10 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T143319442153Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855203840, + "active_bytes_current_delta": 31623306240, + "active_bytes_peak": 92107182080, + "active_bytes_peak_delta": 56640403456, + "after": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 92107182080, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 92107182080, + "device": 4, + "inactive_split_bytes_current": 7530777600, + "inactive_split_bytes_peak": 12127448576, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 97804877824, + "reserved_bytes_peak": 97804877824, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:33:19.441745Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855203840, + "allocated_bytes_current_delta": 31623306240, + "allocated_bytes_peak": 92107182080, + "allocated_bytes_peak_delta": 56640403456, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:32:11.736265Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7530777600, + "inactive_split_bytes_current_delta": 7398401024, + "inactive_split_bytes_peak": 12127448576, + "inactive_split_bytes_peak_delta": 11974039040, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 97804877824, + "reserved_bytes_current_delta": 58560872448, + "reserved_bytes_peak": 97804877824, + "reserved_bytes_peak_delta": 58560872448, + "segment_count_current": 443, + "segment_count_current_delta": 372, + "segment_count_peak": 443, + "segment_count_peak_delta": 372, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T143319442153Z.pickle", + "timestamp_utc": "2026-08-01T14:33:19.557908Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T143319442153Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T143319442153Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2fa20654faada19ff4a91d15ad6f01a8f92062d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T143319442153Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec30bdb749da977c18e02f4755e7f345bbc4b6b2b704eecba172d713b3d86ccd +size 5382366 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T143320684155Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T143320684155Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a70f29fce5d2c79914abfadab6cce4baae6c4c5b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T143320684155Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855203840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115753033728, + "active_bytes_peak_delta": 23645851648, + "after": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115753033728, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115753033728, + "device": 4, + "inactive_split_bytes_current": 7595789312, + "inactive_split_bytes_peak": 12127448576, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:20.683699Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855203840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115753033728, + "allocated_bytes_peak_delta": 23645851648, + "before": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 92107182080, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 92107182080, + "device": 4, + "inactive_split_bytes_current": 7530777600, + "inactive_split_bytes_peak": 12127448576, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 97804877824, + "reserved_bytes_peak": 97804877824, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:33:19.577107Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7595789312, + "inactive_split_bytes_current_delta": 65011712, + "inactive_split_bytes_peak": 12127448576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117945925632, + "reserved_bytes_current_delta": 20141047808, + "reserved_bytes_peak": 117945925632, + "reserved_bytes_peak_delta": 20141047808, + "segment_count_current": 499, + "segment_count_current_delta": 56, + "segment_count_peak": 499, + "segment_count_peak_delta": 56, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T143320684155Z.pickle", + "timestamp_utc": "2026-08-01T14:33:20.799501Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T143320684155Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T143320684155Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b2e7a3dc1fc3eb6235461c04caea62c3a03ee234 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T143320684155Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d4fb64c9a10961d5977dddfcb349c7bdf69e45f36f30446213d323b4edeab8b +size 5602281 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T143321898551Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T143321898551Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f5913dba61fb63aa6d1c91a13e8f81866ce5e2d3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T143321898551Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855203840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115756109824, + "active_bytes_peak_delta": 3076096, + "after": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115756109824, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115756109824, + "device": 4, + "inactive_split_bytes_current": 7469960192, + "inactive_split_bytes_peak": 12127448576, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:21.898090Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855203840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115756109824, + "allocated_bytes_peak_delta": 3076096, + "before": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115753033728, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115753033728, + "device": 4, + "inactive_split_bytes_current": 7595789312, + "inactive_split_bytes_peak": 12127448576, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:20.816021Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7469960192, + "inactive_split_bytes_current_delta": -125829120, + "inactive_split_bytes_peak": 12127448576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117945925632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117945925632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T143321898551Z.pickle", + "timestamp_utc": "2026-08-01T14:33:22.007270Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T143321898551Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T143321898551Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7716f7e3992bb96fa98edea05439b958072fc53c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T143321898551Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d1a1ab253cff427fa8978a66bda426f311edd0571c2f55d7c97e2f4701fd4ba +size 5603949 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T143323121718Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T143323121718Z.json new file mode 100644 index 0000000000000000000000000000000000000000..105b5609c3b383bfbac3f643ea1e9b918a3f28ed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T143323121718Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855203840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115756109824, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115756109824, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115756109824, + "device": 4, + "inactive_split_bytes_current": 9118321664, + "inactive_split_bytes_peak": 12127448576, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:23.121297Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855203840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115756109824, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115756109824, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115756109824, + "device": 4, + "inactive_split_bytes_current": 7469960192, + "inactive_split_bytes_peak": 12127448576, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:22.022719Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9118321664, + "inactive_split_bytes_current_delta": 1648361472, + "inactive_split_bytes_peak": 12127448576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117945925632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117945925632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T143323121718Z.pickle", + "timestamp_utc": "2026-08-01T14:33:23.232504Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T143323121718Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T143323121718Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..96aaeebbde8aa676aaaff05d5ea6368acfbd8c9b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T143323121718Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cbc565e36cca63859f773cbfcad89fa130692c1b2755319d20105b6113c7c7f +size 5602138 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T143324705435Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T143324705435Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7eee601ae246c7744a5f3db753d797cb3d39b646 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T143324705435Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855203840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115758731264, + "active_bytes_peak_delta": 2621440, + "after": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115758731264, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115758731264, + "device": 4, + "inactive_split_bytes_current": 8826817536, + "inactive_split_bytes_peak": 12127448576, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:24.705013Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855203840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115758731264, + "allocated_bytes_peak_delta": 2621440, + "before": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115756109824, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115756109824, + "device": 4, + "inactive_split_bytes_current": 9118321664, + "inactive_split_bytes_peak": 12127448576, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:23.250077Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8826817536, + "inactive_split_bytes_current_delta": -291504128, + "inactive_split_bytes_peak": 12127448576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117945925632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117945925632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T143324705435Z.pickle", + "timestamp_utc": "2026-08-01T14:33:24.813756Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T143324705435Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T143324705435Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a7067530aedf7ece48a006982111abb3fb41020a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T143324705435Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d69be2b8add967de122cd9d8005b10d011efb015dd0a7521400d07fb0894c50 +size 5601745 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T143325896012Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T143325896012Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3f687222aee6ed59e7ef66962fc515e361b09c42 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T143325896012Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855203840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115758731264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115758731264, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115758731264, + "device": 4, + "inactive_split_bytes_current": 8300432384, + "inactive_split_bytes_peak": 12127448576, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:25.895610Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855203840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115758731264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115758731264, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115758731264, + "device": 4, + "inactive_split_bytes_current": 8826817536, + "inactive_split_bytes_peak": 12127448576, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:24.828375Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8300432384, + "inactive_split_bytes_current_delta": -526385152, + "inactive_split_bytes_peak": 12127448576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117945925632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117945925632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T143325896012Z.pickle", + "timestamp_utc": "2026-08-01T14:33:25.999555Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T143325896012Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T143325896012Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b90dafdc22701995e0854d08fa97868e4a29d2af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T143325896012Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b20d71545e75cc5b6f2b8991ff18096c2b1aeb662e8fe78b2686ca6b73cc2fdf +size 5601773 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T143327090345Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T143327090345Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f26c3d62d092bdcbde13db8493ade7132dbfdc5a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T143327090345Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855203840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115758731264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115758731264, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115758731264, + "device": 4, + "inactive_split_bytes_current": 8239614976, + "inactive_split_bytes_peak": 12127448576, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:27.089929Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855203840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115758731264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115758731264, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115758731264, + "device": 4, + "inactive_split_bytes_current": 8300432384, + "inactive_split_bytes_peak": 12127448576, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:26.014117Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8239614976, + "inactive_split_bytes_current_delta": -60817408, + "inactive_split_bytes_peak": 12127448576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117945925632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117945925632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T143327090345Z.pickle", + "timestamp_utc": "2026-08-01T14:33:27.193147Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T143327090345Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T143327090345Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c2e50651fc3fcea85d908616097bc7bb3f9fb5f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T143327090345Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d01a2ccd056af0e5bcb68d6497b52d4a3e4f95075b3dccc75cd07893c79127ce +size 5601468 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T143328276421Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T143328276421Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3b3e1ed07090f98e1faba627abc92e98bfe1045b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T143328276421Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855203840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115758731264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115758731264, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115758731264, + "device": 4, + "inactive_split_bytes_current": 7830670336, + "inactive_split_bytes_peak": 12127448576, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:28.275998Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855203840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115758731264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115758731264, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115758731264, + "device": 4, + "inactive_split_bytes_current": 8239614976, + "inactive_split_bytes_peak": 12127448576, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:27.207673Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7830670336, + "inactive_split_bytes_current_delta": -408944640, + "inactive_split_bytes_peak": 12127448576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117945925632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117945925632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T143328276421Z.pickle", + "timestamp_utc": "2026-08-01T14:33:28.379305Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T143328276421Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T143328276421Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e66073a712e07beccc9faa8577c4c5fc02f2686d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T143328276421Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51e763978b0be19845e6fb0e5277604a1d92a7a156c699facce87bceded2ae01 +size 5601355 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T143329471081Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T143329471081Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5d5f9349aec8b718cba67a2e577aa8a25622f765 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T143329471081Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855203840, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115758731264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115758731264, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115758731264, + "device": 4, + "inactive_split_bytes_current": 7941819392, + "inactive_split_bytes_peak": 12127448576, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:29.470674Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855203840, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115758731264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115758731264, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115758731264, + "device": 4, + "inactive_split_bytes_current": 7830670336, + "inactive_split_bytes_peak": 12127448576, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:28.393585Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7941819392, + "inactive_split_bytes_current_delta": 111149056, + "inactive_split_bytes_peak": 12127448576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117945925632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117945925632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T143329471081Z.pickle", + "timestamp_utc": "2026-08-01T14:33:29.574755Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T143329471081Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T143329471081Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de6030809c3876953f6eec8113eeff43c19f95a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T143329471081Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93055094f3b5443c124d9ee8c80256bcc9d26acd61df2bc601be092ec195fecf +size 5601665 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T143425847930Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T143425847930Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1d165c809c527857dcf2dbfc83434b84aa20ec95 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T143425847930Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804560896, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115758731264, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66804560896, + "active_bytes_peak": 115758731264, + "allocated_bytes_current": 66804560896, + "allocated_bytes_peak": 115758731264, + "device": 4, + "inactive_split_bytes_current": 5966613504, + "inactive_split_bytes_peak": 12127448576, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:34:25.847470Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804560896, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115758731264, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855203840, + "active_bytes_peak": 115758731264, + "allocated_bytes_current": 66855203840, + "allocated_bytes_peak": 115758731264, + "device": 4, + "inactive_split_bytes_current": 7941819392, + "inactive_split_bytes_peak": 12127448576, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 117945925632, + "reserved_bytes_peak": 117945925632, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:29.590811Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5966613504, + "inactive_split_bytes_current_delta": -1975205888, + "inactive_split_bytes_peak": 12127448576, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 117945925632, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 117945925632, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T143425847930Z.pickle", + "timestamp_utc": "2026-08-01T14:34:26.047276Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T143425847930Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T143425847930Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..94b430a3d90a5b8ec61f1270e7d7b06a3dd3b2bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T143425847930Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:420b8152ba8ef79e161cf054a73b20d96c40e1c7cd2759adb30412414dbb1fc0 +size 9990956 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T143319444291Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T143319444291Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b198bb3e5bb4e24513c08d3ef2b06216dfd84230 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T143319444291Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 92103927296, + "active_bytes_peak_delta": 56637591040, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92103927296, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92103927296, + "device": 5, + "inactive_split_bytes_current": 7876365312, + "inactive_split_bytes_peak": 11258790912, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 98333360128, + "reserved_bytes_peak": 98333360128, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T14:33:19.443631Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 92103927296, + "allocated_bytes_peak_delta": 56637591040, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:32:11.736394Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7876365312, + "inactive_split_bytes_current_delta": 7743546368, + "inactive_split_bytes_peak": 11258790912, + "inactive_split_bytes_peak_delta": 11098404864, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 98333360128, + "reserved_bytes_current_delta": 59089354752, + "reserved_bytes_peak": 98333360128, + "reserved_bytes_peak_delta": 59089354752, + "segment_count_current": 448, + "segment_count_current_delta": 377, + "segment_count_peak": 448, + "segment_count_peak_delta": 377, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T143319444291Z.pickle", + "timestamp_utc": "2026-08-01T14:33:19.555063Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T143319444291Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T143319444291Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca2b4db4c49956d8697759c360f715a7490b5020 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T143319444291Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78d9dd123998007cbc4e2b4e82ca15b4d921b31d9a4bdfa98311a33c43932d0c +size 5377163 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T143320684061Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T143320684061Z.json new file mode 100644 index 0000000000000000000000000000000000000000..32e9c2e7f2fa18152f579c7c575b07192899e0a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T143320684061Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115751597056, + "active_bytes_peak_delta": 23647669760, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115751597056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115751597056, + "device": 5, + "inactive_split_bytes_current": 8943815680, + "inactive_split_bytes_peak": 11499950080, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:20.683649Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115751597056, + "allocated_bytes_peak_delta": 23647669760, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92103927296, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92103927296, + "device": 5, + "inactive_split_bytes_current": 7876365312, + "inactive_split_bytes_peak": 11258790912, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 98333360128, + "reserved_bytes_peak": 98333360128, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T14:33:19.574908Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8943815680, + "inactive_split_bytes_current_delta": 1067450368, + "inactive_split_bytes_peak": 11499950080, + "inactive_split_bytes_peak_delta": 241159168, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118021423104, + "reserved_bytes_current_delta": 19688062976, + "reserved_bytes_peak": 118021423104, + "reserved_bytes_peak_delta": 19688062976, + "segment_count_current": 499, + "segment_count_current_delta": 51, + "segment_count_peak": 499, + "segment_count_peak_delta": 51, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T143320684061Z.pickle", + "timestamp_utc": "2026-08-01T14:33:20.796504Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T143320684061Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T143320684061Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9ea4450900608f34599dc3423fe9a0f08a82bc1a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T143320684061Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea502665d5708e74531a549fab355309c0162b5634b02717d24768845b2a6619 +size 5568970 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T143321898489Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T143321898489Z.json new file mode 100644 index 0000000000000000000000000000000000000000..434a03ab5c23f0f021c595299bb4ee7ff7ec96eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T143321898489Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115755952128, + "active_bytes_peak_delta": 4355072, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115755952128, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115755952128, + "device": 5, + "inactive_split_bytes_current": 7714884608, + "inactive_split_bytes_peak": 11499950080, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:21.898082Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115755952128, + "allocated_bytes_peak_delta": 4355072, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115751597056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115751597056, + "device": 5, + "inactive_split_bytes_current": 8943815680, + "inactive_split_bytes_peak": 11499950080, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:20.812785Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7714884608, + "inactive_split_bytes_current_delta": -1228931072, + "inactive_split_bytes_peak": 11499950080, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118021423104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118021423104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T143321898489Z.pickle", + "timestamp_utc": "2026-08-01T14:33:22.002815Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T143321898489Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T143321898489Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6171e4b5f1c2ac9e3be3cae8677495fc38eb9e03 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T143321898489Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f6071aa0689c1c8e71a0c8f055c925a3f874b05ee16c149757fe618685a2f44 +size 5598608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T143323121751Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T143323121751Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06a2b504b0a78596069115955cb57a52c4c4b8af --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T143323121751Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115762081792, + "active_bytes_peak_delta": 6129664, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 10237758464, + "inactive_split_bytes_peak": 11677007872, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:23.121344Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115762081792, + "allocated_bytes_peak_delta": 6129664, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115755952128, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115755952128, + "device": 5, + "inactive_split_bytes_current": 7714884608, + "inactive_split_bytes_peak": 11499950080, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:22.018129Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10237758464, + "inactive_split_bytes_current_delta": 2522873856, + "inactive_split_bytes_peak": 11677007872, + "inactive_split_bytes_peak_delta": 177057792, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118021423104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118021423104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T143323121751Z.pickle", + "timestamp_utc": "2026-08-01T14:33:23.228260Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T143323121751Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T143323121751Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..26eb7c24e8ef9505206fd9b4a5d3b88227209eb7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T143323121751Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29409c5f28eb79f3ededacebdf7f2556e46393e2e0002250e775b0ebd9843d32 +size 5600199 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T143324705443Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T143324705443Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f12e86c9a566f97d8d40fbeceda9755b0e87184d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T143324705443Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115762081792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 8010583040, + "inactive_split_bytes_peak": 11677007872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:24.705029Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115762081792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 10237758464, + "inactive_split_bytes_peak": 11677007872, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:23.243024Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8010583040, + "inactive_split_bytes_current_delta": -2227175424, + "inactive_split_bytes_peak": 11677007872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118021423104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118021423104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T143324705443Z.pickle", + "timestamp_utc": "2026-08-01T14:33:24.810762Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T143324705443Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T143324705443Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..40a6598b78e0d7d757b9b3be7810156774aa474c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T143324705443Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1a1987cc9e3a3270c46c6c01d636d944488538bcf1b7d07b06b1809c912c0cf +size 5581994 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T143325896012Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T143325896012Z.json new file mode 100644 index 0000000000000000000000000000000000000000..93e8c33dabee61e659bab0e2c41bed498ed47fbe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T143325896012Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115762081792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 7939279872, + "inactive_split_bytes_peak": 11677007872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:25.895623Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115762081792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 8010583040, + "inactive_split_bytes_peak": 11677007872, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:24.825394Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7939279872, + "inactive_split_bytes_current_delta": -71303168, + "inactive_split_bytes_peak": 11677007872, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118021423104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118021423104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T143325896012Z.pickle", + "timestamp_utc": "2026-08-01T14:33:25.997698Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T143325896012Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T143325896012Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7dd2f52a8ce2c257f7a51c1d983a392616da1fbb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T143325896012Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:012002b6ba048c5ef57295a45620616af2ffee2b63ac9ada262ee7e658c8e8f1 +size 5600440 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T143327090340Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T143327090340Z.json new file mode 100644 index 0000000000000000000000000000000000000000..542fa1c468efec4744171b9c4344dcd28b3185e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T143327090340Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115762081792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 8771849216, + "inactive_split_bytes_peak": 11829364736, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:27.089944Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115762081792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 7939279872, + "inactive_split_bytes_peak": 11677007872, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:26.012322Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8771849216, + "inactive_split_bytes_current_delta": 832569344, + "inactive_split_bytes_peak": 11829364736, + "inactive_split_bytes_peak_delta": 152356864, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118021423104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118021423104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T143327090340Z.pickle", + "timestamp_utc": "2026-08-01T14:33:27.191835Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T143327090340Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T143327090340Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ec0c707b38d1db174880fdfc7e287da2de90c23 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T143327090340Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6d80f358afae35548898564875e31125e39f333cbcb0d9649da081e625523e2 +size 5591878 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T143328276377Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T143328276377Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7cfdc3f4fde2e59dbd3953aeda8c27bd788d7624 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T143328276377Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115762081792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 7295454208, + "inactive_split_bytes_peak": 11829364736, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:28.275972Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115762081792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 8771849216, + "inactive_split_bytes_peak": 11829364736, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:27.206253Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7295454208, + "inactive_split_bytes_current_delta": -1476395008, + "inactive_split_bytes_peak": 11829364736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118021423104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118021423104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T143328276377Z.pickle", + "timestamp_utc": "2026-08-01T14:33:28.376502Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T143328276377Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T143328276377Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0cba6fe894e4ca2896924467593d022f82873371 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T143328276377Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dde876cd87fb284bace9877c16ccb0409b3accc5c33fc79a00442c5d33abbeb9 +size 5597018 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T143329471084Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T143329471084Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a1ab82b35817db6c30793925f3388934d8c35bac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T143329471084Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115762081792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 8172063744, + "inactive_split_bytes_peak": 11829364736, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:29.470675Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115762081792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 7295454208, + "inactive_split_bytes_peak": 11829364736, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:28.390504Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8172063744, + "inactive_split_bytes_current_delta": 876609536, + "inactive_split_bytes_peak": 11829364736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118021423104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118021423104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T143329471084Z.pickle", + "timestamp_utc": "2026-08-01T14:33:29.571954Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T143329471084Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T143329471084Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c4b6d0554601b95b08383dc898ba628b0356b642 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T143329471084Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0af415685921014112b1d56e1c2e47dd890cc3e131912f33ca92917bd0005a2 +size 5599419 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T143425848015Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T143425848015Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dcfb34093cfeaaf8eb4fd4ae2b324d48f78561da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T143425848015Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115762081792, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 6234606592, + "inactive_split_bytes_peak": 11829364736, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:34:25.847480Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115762081792, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115762081792, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115762081792, + "device": 5, + "inactive_split_bytes_current": 8172063744, + "inactive_split_bytes_peak": 11829364736, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118021423104, + "reserved_bytes_peak": 118021423104, + "segment_count_current": 499, + "segment_count_peak": 499, + "timestamp_utc": "2026-08-01T14:33:29.586261Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6234606592, + "inactive_split_bytes_current_delta": -1937457152, + "inactive_split_bytes_peak": 11829364736, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118021423104, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118021423104, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 499, + "segment_count_current_delta": 0, + "segment_count_peak": 499, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T143425848015Z.pickle", + "timestamp_utc": "2026-08-01T14:34:26.048836Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T143425848015Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T143425848015Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fcf521d8459d0aa004fa12fd00e49359036958e7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T143425848015Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42a9eb949f5351227609814b2dc3cdc25c90ecbc9382a45137daa1e73418c3a2 +size 9186220 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T143319442342Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T143319442342Z.json new file mode 100644 index 0000000000000000000000000000000000000000..21da24bce1efc496d54a90954b104514bf7fcc8a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T143319442342Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 92103927296, + "active_bytes_peak_delta": 56637591040, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92103927296, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92103927296, + "device": 6, + "inactive_split_bytes_current": 7526140928, + "inactive_split_bytes_peak": 11413980160, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98333360128, + "reserved_bytes_peak": 98333360128, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T14:33:19.441747Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 92103927296, + "allocated_bytes_peak_delta": 56637591040, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:32:11.735715Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7526140928, + "inactive_split_bytes_current_delta": 7393321984, + "inactive_split_bytes_peak": 11413980160, + "inactive_split_bytes_peak_delta": 11253594112, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98333360128, + "reserved_bytes_current_delta": 59089354752, + "reserved_bytes_peak": 98333360128, + "reserved_bytes_peak_delta": 59089354752, + "segment_count_current": 448, + "segment_count_current_delta": 377, + "segment_count_peak": 448, + "segment_count_peak_delta": 377, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T143319442342Z.pickle", + "timestamp_utc": "2026-08-01T14:33:19.553433Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T143319442342Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T143319442342Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e8c3088a870a87f446aa4f8058e2d551bfd8a5c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T143319442342Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1052ad108cf959c77b1e9a98ecdaac67cba919df232c676bfc02a73b03ef8d8 +size 5377073 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T143320684082Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T143320684082Z.json new file mode 100644 index 0000000000000000000000000000000000000000..591828c44b8f3c29d29ac38e861a505c98b6a69f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T143320684082Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115750526976, + "active_bytes_peak_delta": 23646599680, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115750526976, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115750526976, + "device": 6, + "inactive_split_bytes_current": 7943474176, + "inactive_split_bytes_peak": 11594764288, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:20.683680Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115750526976, + "allocated_bytes_peak_delta": 23646599680, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92103927296, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92103927296, + "device": 6, + "inactive_split_bytes_current": 7526140928, + "inactive_split_bytes_peak": 11413980160, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98333360128, + "reserved_bytes_peak": 98333360128, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T14:33:19.570920Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7943474176, + "inactive_split_bytes_current_delta": 417333248, + "inactive_split_bytes_peak": 11594764288, + "inactive_split_bytes_peak_delta": 180784128, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 19746783232, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 19746783232, + "segment_count_current": 500, + "segment_count_current_delta": 52, + "segment_count_peak": 500, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T143320684082Z.pickle", + "timestamp_utc": "2026-08-01T14:33:20.793279Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T143320684082Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T143320684082Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b215ec9603e9af5105ee5498bf51ef568394260d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T143320684082Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29985e009f90f938edebafde43858991f58668de619a07fe8a8a3d01525a1ca3 +size 5601623 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T143321898486Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T143321898486Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ed0f8a232f33613f879d576233075ad6b94d103 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T143321898486Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115755810816, + "active_bytes_peak_delta": 5283840, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115755810816, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115755810816, + "device": 6, + "inactive_split_bytes_current": 9113684992, + "inactive_split_bytes_peak": 11594764288, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:21.898077Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115755810816, + "allocated_bytes_peak_delta": 5283840, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115750526976, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115750526976, + "device": 6, + "inactive_split_bytes_current": 7943474176, + "inactive_split_bytes_peak": 11594764288, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:20.805326Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9113684992, + "inactive_split_bytes_current_delta": 1170210816, + "inactive_split_bytes_peak": 11594764288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T143321898486Z.pickle", + "timestamp_utc": "2026-08-01T14:33:22.004030Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T143321898486Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T143321898486Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dd84187b05b788e1cd2f8bef1ec035e8be197e96 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T143321898486Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:050b2d4fa30fd7cb50cf77602739a7ac7bcc56db62b5b1b8303db5819a0c4023 +size 5601251 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T143323121689Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T143323121689Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ee332c49d62a9ab420cc19baac2cef0eed11e44e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T143323121689Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115757867008, + "active_bytes_peak_delta": 2056192, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115757867008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115757867008, + "device": 6, + "inactive_split_bytes_current": 9124170752, + "inactive_split_bytes_peak": 11594764288, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:23.121287Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115757867008, + "allocated_bytes_peak_delta": 2056192, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115755810816, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115755810816, + "device": 6, + "inactive_split_bytes_current": 9113684992, + "inactive_split_bytes_peak": 11594764288, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:22.017307Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9124170752, + "inactive_split_bytes_current_delta": 10485760, + "inactive_split_bytes_peak": 11594764288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T143323121689Z.pickle", + "timestamp_utc": "2026-08-01T14:33:23.218207Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T143323121689Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T143323121689Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..14424ded81977e86b1b534206cef35b78d9aae1d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T143323121689Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5520ccee24835f0de2ce2289160d5a96aebf0d6550cc8b9a2aa83a44a13ce20e +size 5600140 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T143324705463Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T143324705463Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bc6a395d6732aff23f86c874ce47031361b025ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T143324705463Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759722496, + "active_bytes_peak_delta": 1855488, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 6, + "inactive_split_bytes_current": 8306281472, + "inactive_split_bytes_peak": 11594764288, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:24.705033Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759722496, + "allocated_bytes_peak_delta": 1855488, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115757867008, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115757867008, + "device": 6, + "inactive_split_bytes_current": 9124170752, + "inactive_split_bytes_peak": 11594764288, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:23.229300Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8306281472, + "inactive_split_bytes_current_delta": -817889280, + "inactive_split_bytes_peak": 11594764288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T143324705463Z.pickle", + "timestamp_utc": "2026-08-01T14:33:24.807714Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T143324705463Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T143324705463Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..97a54750bf7c0b4a73acfe03d26aa5130e827db5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T143324705463Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a45915f2c4f6f0e87d3086a2ecd82c1be06e91c23c52e9cea82cbe55c0b5a2e8 +size 5600465 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T143325896022Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T143325896022Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f1522103b7f0fa2f7da16810719ca19412ce1f29 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T143325896022Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 6, + "inactive_split_bytes_current": 9063353344, + "inactive_split_bytes_peak": 11594764288, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:25.895620Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 6, + "inactive_split_bytes_current": 8306281472, + "inactive_split_bytes_peak": 11594764288, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:24.819452Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9063353344, + "inactive_split_bytes_current_delta": 757071872, + "inactive_split_bytes_peak": 11594764288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T143325896022Z.pickle", + "timestamp_utc": "2026-08-01T14:33:25.992801Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T143325896022Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T143325896022Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c8507c53b2f596d08d82ddefbacbc6bb0ad90f1e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T143325896022Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d708d10e7859691da5a54ff81c4f9c7a56ac5bc8ee4d9139c7f517c7fb0a2ed2 +size 5600114 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T143327090370Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T143327090370Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bfa0f48c17cf4d75a32ebfd26d8fa6f6ca7f064f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T143327090370Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 6, + "inactive_split_bytes_current": 7532432384, + "inactive_split_bytes_peak": 11594764288, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:27.089936Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 6, + "inactive_split_bytes_current": 9063353344, + "inactive_split_bytes_peak": 11594764288, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:26.004033Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7532432384, + "inactive_split_bytes_current_delta": -1530920960, + "inactive_split_bytes_peak": 11594764288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T143327090370Z.pickle", + "timestamp_utc": "2026-08-01T14:33:27.186141Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T143327090370Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T143327090370Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..67d6f1fc7e58a88ba517b8b903df1b5596ce5576 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T143327090370Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a133982c3aa23101a4ad4b0ebc12bab0c9cb78aed766ec064e85b15f9c13bbc0 +size 5582050 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T143328276384Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T143328276384Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2807a6df325fb25a7214465c87c86a97fa70859 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T143328276384Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 6, + "inactive_split_bytes_current": 9002535936, + "inactive_split_bytes_peak": 11594764288, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:28.275987Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 6, + "inactive_split_bytes_current": 7532432384, + "inactive_split_bytes_peak": 11594764288, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:27.197260Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9002535936, + "inactive_split_bytes_current_delta": 1470103552, + "inactive_split_bytes_peak": 11594764288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T143328276384Z.pickle", + "timestamp_utc": "2026-08-01T14:33:28.373088Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T143328276384Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T143328276384Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3356ed5cef68c808a564a5f2f28047558ee13551 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T143328276384Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d63d3545fbffa8e36af7b47d404716605c06571d12f1dfebb5494bec31f79156 +size 5600921 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T143329471092Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T143329471092Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0143f5e0bdb6ab5e8a656c6b5bc5e7c971f2d69d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T143329471092Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115759722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 6, + "inactive_split_bytes_current": 9470200832, + "inactive_split_bytes_peak": 11594764288, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:29.470683Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115759722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 6, + "inactive_split_bytes_current": 9002535936, + "inactive_split_bytes_peak": 11594764288, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:28.386246Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9470200832, + "inactive_split_bytes_current_delta": 467664896, + "inactive_split_bytes_peak": 11594764288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T143329471092Z.pickle", + "timestamp_utc": "2026-08-01T14:33:29.568799Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T143329471092Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T143329471092Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e26e22bf85eb0fe139167d3d74551abb8fae07d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T143329471092Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:652990c15d1d8ebc032d23142f1e4acc4954412c2069e282714a71089ea90caf +size 5600203 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T143425847926Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T143425847926Z.json new file mode 100644 index 0000000000000000000000000000000000000000..37c7cec51de0d5efed04f39a816bee20f292ffa8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T143425847926Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115759722496, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 115759722496, + "device": 6, + "inactive_split_bytes_current": 5999725568, + "inactive_split_bytes_peak": 11594764288, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:34:25.847461Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115759722496, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115759722496, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115759722496, + "device": 6, + "inactive_split_bytes_current": 9470200832, + "inactive_split_bytes_peak": 11594764288, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118080143360, + "reserved_bytes_peak": 118080143360, + "segment_count_current": 500, + "segment_count_peak": 500, + "timestamp_utc": "2026-08-01T14:33:29.580312Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5999725568, + "inactive_split_bytes_current_delta": -3470475264, + "inactive_split_bytes_peak": 11594764288, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118080143360, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118080143360, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 500, + "segment_count_current_delta": 0, + "segment_count_peak": 500, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T143425847926Z.pickle", + "timestamp_utc": "2026-08-01T14:34:26.043295Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T143425847926Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T143425847926Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4f032e431fc5a918f358f6590fad11b83dd455b5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T143425847926Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e231eb5f417c3fec095a2c4aaa478babb7285fe193159524738fb24f27c5e81 +size 9186657 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T143319442328Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T143319442328Z.json new file mode 100644 index 0000000000000000000000000000000000000000..221ecd6ad87f73461dde25f774d4d3166dc57d49 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T143319442328Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 31622094336, + "active_bytes_peak": 92103927296, + "active_bytes_peak_delta": 56637591040, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 92103927296, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 92103927296, + "device": 7, + "inactive_split_bytes_current": 7524043264, + "inactive_split_bytes_peak": 11416077312, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 98333360128, + "reserved_bytes_peak": 98333360128, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T14:33:19.441821Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 31622094336, + "allocated_bytes_peak": 92103927296, + "allocated_bytes_peak_delta": 56637591040, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:32:11.738990Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7524043264, + "inactive_split_bytes_current_delta": 7391224320, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 11255691264, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 98333360128, + "reserved_bytes_current_delta": 59089354752, + "reserved_bytes_peak": 98333360128, + "reserved_bytes_peak_delta": 59089354752, + "segment_count_current": 448, + "segment_count_current_delta": 377, + "segment_count_peak": 448, + "segment_count_peak_delta": 377, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T143319442328Z.pickle", + "timestamp_utc": "2026-08-01T14:33:19.554741Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T143319442328Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T143319442328Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..33f3c3cd234c80e7977c8849b3e0156fce63d59e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T143319442328Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84c1f60acad44e08660f077286e9495f6f63fc454b90ab0d4799efc6b56e4208 +size 5378526 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T143320684094Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T143320684094Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cfc3402ab3cd58e9ab91b2a9b38ddb0e10bc7a40 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T143320684094Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115756294656, + "active_bytes_peak_delta": 23652367360, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115756294656, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115756294656, + "device": 7, + "inactive_split_bytes_current": 8650213888, + "inactive_split_bytes_peak": 11416077312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:20.683665Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115756294656, + "allocated_bytes_peak_delta": 23652367360, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 92103927296, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 92103927296, + "device": 7, + "inactive_split_bytes_current": 7524043264, + "inactive_split_bytes_peak": 11416077312, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 98333360128, + "reserved_bytes_peak": 98333360128, + "segment_count_current": 448, + "segment_count_peak": 448, + "timestamp_utc": "2026-08-01T14:33:19.573878Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8650213888, + "inactive_split_bytes_current_delta": 1126170624, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 19805503488, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 19805503488, + "segment_count_current": 501, + "segment_count_current_delta": 53, + "segment_count_peak": 501, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T143320684094Z.pickle", + "timestamp_utc": "2026-08-01T14:33:20.794480Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T143320684094Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T143320684094Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c3ac17664b8c0328f4e69b697fd00c9f6ac8ec0f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T143320684094Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2269c05af12fcc90b03bd75cd1c243baa61410880633c8e3e8bcaa145a64e1d +size 5572491 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T143321898487Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T143321898487Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b576c001fa1901161d1d5e19eafc1d187caea761 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T143321898487Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115756294656, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115756294656, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115756294656, + "device": 7, + "inactive_split_bytes_current": 9113684480, + "inactive_split_bytes_peak": 11416077312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:21.898063Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115756294656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115756294656, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115756294656, + "device": 7, + "inactive_split_bytes_current": 8650213888, + "inactive_split_bytes_peak": 11416077312, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:20.808060Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9113684480, + "inactive_split_bytes_current_delta": 463470592, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T143321898487Z.pickle", + "timestamp_utc": "2026-08-01T14:33:22.005275Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T143321898487Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T143321898487Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..df76a545a466e093e225a0cf5d9f29a4e642a002 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T143321898487Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:854319c45657ba5ca9a816b13c36efeeff20946044afd81bf7f4ed4b0daabb6f +size 5588907 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T143323121704Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T143323121704Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fb639148d6e0a4475af3335acf692748813cdd4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T143323121704Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115756294656, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115756294656, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115756294656, + "device": 7, + "inactive_split_bytes_current": 8241269248, + "inactive_split_bytes_peak": 11416077312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:23.121284Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115756294656, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115756294656, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115756294656, + "device": 7, + "inactive_split_bytes_current": 9113684480, + "inactive_split_bytes_peak": 11416077312, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:22.021814Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8241269248, + "inactive_split_bytes_current_delta": -872415232, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T143323121704Z.pickle", + "timestamp_utc": "2026-08-01T14:33:23.223561Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T143323121704Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T143323121704Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76040a2d3678cf23f781f175a094ba8b3be52b3b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T143323121704Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dc3fa5c3cb02f131eebda41b8d51d530315cb895d465ac019dc42105916218d +size 5581359 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T143324705532Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T143324705532Z.json new file mode 100644 index 0000000000000000000000000000000000000000..12b97ad4030cc5ab92a7e278ae67be933eae75ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T143324705532Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760992768, + "active_bytes_peak_delta": 4698112, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115760992768, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115760992768, + "device": 7, + "inactive_split_bytes_current": 7895239168, + "inactive_split_bytes_peak": 11416077312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:24.705069Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760992768, + "allocated_bytes_peak_delta": 4698112, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115756294656, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115756294656, + "device": 7, + "inactive_split_bytes_current": 8241269248, + "inactive_split_bytes_peak": 11416077312, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:23.236365Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7895239168, + "inactive_split_bytes_current_delta": -346030080, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T143324705532Z.pickle", + "timestamp_utc": "2026-08-01T14:33:24.809719Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T143324705532Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T143324705532Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..72d5b22ee8e81d2d3779468f06bad07ff372595f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T143324705532Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:448b858c21f35692028ed1a0f9b6bb36fdf35e486f9d2dde4f9423bfe0a87daa +size 5592864 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T143325896122Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T143325896122Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3e63de6bb94734e936a3e2e9c73eeff5abe1d1c2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T143325896122Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760992768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115760992768, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115760992768, + "device": 7, + "inactive_split_bytes_current": 8939620864, + "inactive_split_bytes_peak": 11416077312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:25.895633Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760992768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115760992768, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115760992768, + "device": 7, + "inactive_split_bytes_current": 7895239168, + "inactive_split_bytes_peak": 11416077312, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:24.822776Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8939620864, + "inactive_split_bytes_current_delta": 1044381696, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T143325896122Z.pickle", + "timestamp_utc": "2026-08-01T14:33:25.994296Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T143325896122Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T143325896122Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b565595081ff2cbb6a2f81db4545eab53286f9ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T143325896122Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:295ff8cde5b53e03ba911770ce1c14fc47614ca591bbc4ad48ded33fc29a7011 +size 5590981 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T143327090402Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T143327090402Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f3e792c03bd7290337468d4b4625d1810ab2cb6b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T143327090402Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760992768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115760992768, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115760992768, + "device": 7, + "inactive_split_bytes_current": 7532431872, + "inactive_split_bytes_peak": 11416077312, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:27.089977Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760992768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115760992768, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115760992768, + "device": 7, + "inactive_split_bytes_current": 8939620864, + "inactive_split_bytes_peak": 11416077312, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:26.006906Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7532431872, + "inactive_split_bytes_current_delta": -1407188992, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T143327090402Z.pickle", + "timestamp_utc": "2026-08-01T14:33:27.189225Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T143327090402Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T143327090402Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6edfa94b932d0c38b59f948cea09cc4f7f7837db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T143327090402Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:619e85064466cc72d9d4a7cf328cc4a7beb04056850473bfed38933e5a47fee1 +size 5586706 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T143328276535Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T143328276535Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5a92f8620e39044f62d398723b3ed8558adf407e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T143328276535Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760992768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115760992768, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115760992768, + "device": 7, + "inactive_split_bytes_current": 8948009472, + "inactive_split_bytes_peak": 11416077312, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:28.276018Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760992768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115760992768, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115760992768, + "device": 7, + "inactive_split_bytes_current": 7532431872, + "inactive_split_bytes_peak": 11416077312, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:27.201801Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8948009472, + "inactive_split_bytes_current_delta": 1415577600, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T143328276535Z.pickle", + "timestamp_utc": "2026-08-01T14:33:28.375125Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T143328276535Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T143328276535Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c461b11b71fd343df22c6f412d999ab58e7a6470 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T143328276535Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dde45eef89cfc96ff8d37fd3b56edae043817821c70fe81bda4a912043034439 +size 5594233 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T143329471169Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T143329471169Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e3a2563a2f6356a23b240260d1fc46c136030275 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T143329471169Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760992768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115760992768, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115760992768, + "device": 7, + "inactive_split_bytes_current": 8587299328, + "inactive_split_bytes_peak": 11416077312, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:29.470723Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760992768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115760992768, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115760992768, + "device": 7, + "inactive_split_bytes_current": 8948009472, + "inactive_split_bytes_peak": 11416077312, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:28.387690Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8587299328, + "inactive_split_bytes_current_delta": -360710144, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T143329471169Z.pickle", + "timestamp_utc": "2026-08-01T14:33:29.569978Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T143329471169Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T143329471169Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3f05154a7295221ff47e4f52c7793dbe23632256 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T143329471169Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2762ca7eba06d7d3db3b701e603fea7d0390935198b37ca8a886efe3c93ef390 +size 5572295 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T143425847918Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T143425847918Z.json new file mode 100644 index 0000000000000000000000000000000000000000..12c46ed7666c9426a124ce10045d82ce1176eefc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T143425847918Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906624, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115760992768, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906624, + "active_bytes_peak": 115760992768, + "allocated_bytes_current": 66802906624, + "allocated_bytes_peak": 115760992768, + "device": 7, + "inactive_split_bytes_current": 6234606080, + "inactive_split_bytes_peak": 11416077312, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:34:25.847473Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906624, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115760992768, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115760992768, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115760992768, + "device": 7, + "inactive_split_bytes_current": 8587299328, + "inactive_split_bytes_peak": 11416077312, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118138863616, + "reserved_bytes_peak": 118138863616, + "segment_count_current": 501, + "segment_count_peak": 501, + "timestamp_utc": "2026-08-01T14:33:29.582479Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6234606080, + "inactive_split_bytes_current_delta": -2352693248, + "inactive_split_bytes_peak": 11416077312, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118138863616, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118138863616, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 501, + "segment_count_current_delta": 0, + "segment_count_peak": 501, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T143425847918Z.pickle", + "timestamp_utc": "2026-08-01T14:34:26.052103Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T143425847918Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T143425847918Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..34eb38b34a6c9d586f7777ace5449c8f7704ac00 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T143425847918Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb3546435231431941bd3867a4c726f5abfd32ff3dd489feeb463a3263d11338 +size 9198355 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..5a370d6b4a24766e2c5a25210bc4a782f5dde3a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 4096 +capacity: 1.50 +expert_dispatch_overlap: off +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6045 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 230 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T143710165762Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T143710165762Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac94630b681f5ab0645524eab5b5a64791c94dec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T143710165762Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441066496, + "active_bytes_current_delta": 37209611264, + "active_bytes_peak": 92002783232, + "active_bytes_peak_delta": 56536446976, + "after": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 92002783232, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 92002783232, + "device": 0, + "inactive_split_bytes_current": 5629611008, + "inactive_split_bytes_peak": 11866870272, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 95258935296, + "reserved_bytes_peak": 95258935296, + "segment_count_current": 368, + "segment_count_peak": 368, + "timestamp_utc": "2026-08-01T14:37:10.165201Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441066496, + "allocated_bytes_current_delta": 37209611264, + "allocated_bytes_peak": 92002783232, + "allocated_bytes_peak_delta": 56536446976, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T14:36:03.160765Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5629611008, + "inactive_split_bytes_current_delta": 5494694912, + "inactive_split_bytes_peak": 11866870272, + "inactive_split_bytes_peak_delta": 11706238464, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 95258935296, + "reserved_bytes_current_delta": 56012832768, + "reserved_bytes_peak": 95258935296, + "reserved_bytes_peak_delta": 56012832768, + "segment_count_current": 368, + "segment_count_current_delta": 296, + "segment_count_peak": 368, + "segment_count_peak_delta": 296, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T143710165762Z.pickle", + "timestamp_utc": "2026-08-01T14:37:10.284490Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T143710165762Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T143710165762Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f81440a20b123bacc232601d03aad140c8c97c85 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T143710165762Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30f375d58efb0d6bc46cfd02258850a350a11ad98686e51bccb21659edbc2327 +size 5935975 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T143711514040Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T143711514040Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b568871c1bd32137a52acc556677c38c7e40ad30 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T143711514040Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442115072, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115649687552, + "active_bytes_peak_delta": 23646904320, + "after": { + "active_bytes_current": 72442115072, + "active_bytes_peak": 115649687552, + "allocated_bytes_current": 72442115072, + "allocated_bytes_peak": 115649687552, + "device": 0, + "inactive_split_bytes_current": 7094471680, + "inactive_split_bytes_peak": 13880519680, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:11.513635Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442115072, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115649687552, + "allocated_bytes_peak_delta": 23646904320, + "before": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 92002783232, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 92002783232, + "device": 0, + "inactive_split_bytes_current": 5629611008, + "inactive_split_bytes_peak": 11866870272, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 95258935296, + "reserved_bytes_peak": 95258935296, + "segment_count_current": 368, + "segment_count_peak": 368, + "timestamp_utc": "2026-08-01T14:37:10.305204Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7094471680, + "inactive_split_bytes_current_delta": 1464860672, + "inactive_split_bytes_peak": 13880519680, + "inactive_split_bytes_peak_delta": 2013649408, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_current_delta": 23173529600, + "reserved_bytes_peak": 118432464896, + "reserved_bytes_peak_delta": 23173529600, + "segment_count_current": 423, + "segment_count_current_delta": 55, + "segment_count_peak": 423, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T143711514040Z.pickle", + "timestamp_utc": "2026-08-01T14:37:11.633757Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T143711514040Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T143711514040Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b6d6ed6e473234f5a82d81cd294654fccca56106 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T143711514040Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7a6c870f90de86a1d4578c65b8b1ed406718c28aaaa1601f75f752fb6f17776 +size 6603097 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T143712862200Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T143712862200Z.json new file mode 100644 index 0000000000000000000000000000000000000000..64a10e03b39f923cf53760ba0da3bce81a5114aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T143712862200Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441066496, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 115651260416, + "active_bytes_peak_delta": 1572864, + "after": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115651260416, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115651260416, + "device": 0, + "inactive_split_bytes_current": 7978421248, + "inactive_split_bytes_peak": 13880519680, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:12.861802Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441066496, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 115651260416, + "allocated_bytes_peak_delta": 1572864, + "before": { + "active_bytes_current": 72442115072, + "active_bytes_peak": 115649687552, + "allocated_bytes_current": 72442115072, + "allocated_bytes_peak": 115649687552, + "device": 0, + "inactive_split_bytes_current": 7094471680, + "inactive_split_bytes_peak": 13880519680, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:11.652212Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7978421248, + "inactive_split_bytes_current_delta": 883949568, + "inactive_split_bytes_peak": 13880519680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118432464896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 423, + "segment_count_current_delta": 0, + "segment_count_peak": 423, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T143712862200Z.pickle", + "timestamp_utc": "2026-08-01T14:37:12.971099Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T143712862200Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T143712862200Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aef8855c1d69992447ca40c74d66ed88ff26b793 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T143712862200Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab02259a4569e2af9c5857fde0763bd6a06cadab8c95ffaf12b69fe06809efb7 +size 6029813 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T143714194938Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T143714194938Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b28d0c15f26dc7a981ea51011ff4dbf666a2555e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T143714194938Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441066496, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115651260416, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115651260416, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115651260416, + "device": 0, + "inactive_split_bytes_current": 7626099712, + "inactive_split_bytes_peak": 14295266304, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:14.194542Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441066496, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115651260416, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115651260416, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115651260416, + "device": 0, + "inactive_split_bytes_current": 7978421248, + "inactive_split_bytes_peak": 13880519680, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:12.986990Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7626099712, + "inactive_split_bytes_current_delta": -352321536, + "inactive_split_bytes_peak": 14295266304, + "inactive_split_bytes_peak_delta": 414746624, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118432464896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 423, + "segment_count_current_delta": 0, + "segment_count_peak": 423, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T143714194938Z.pickle", + "timestamp_utc": "2026-08-01T14:37:14.303309Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T143714194938Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T143714194938Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6a0499513806d01288a8f86787749530da16c5c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T143714194938Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43ee1754eca8ed4ecde78dcf1dc8df127b6f511e4d586abfbae2a343aa1fa727 +size 6607586 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T143715756210Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T143715756210Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db19e6bd29bc8ddb2087d07233fd8c222c8b04bb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T143715756210Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441066496, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115654668288, + "active_bytes_peak_delta": 3407872, + "after": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115654668288, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115654668288, + "device": 0, + "inactive_split_bytes_current": 7099714560, + "inactive_split_bytes_peak": 14295266304, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:15.755809Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441066496, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115654668288, + "allocated_bytes_peak_delta": 3407872, + "before": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115651260416, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115651260416, + "device": 0, + "inactive_split_bytes_current": 7626099712, + "inactive_split_bytes_peak": 14295266304, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:14.319007Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7099714560, + "inactive_split_bytes_current_delta": -526385152, + "inactive_split_bytes_peak": 14295266304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118432464896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 423, + "segment_count_current_delta": 0, + "segment_count_peak": 423, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T143715756210Z.pickle", + "timestamp_utc": "2026-08-01T14:37:15.866017Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T143715756210Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T143715756210Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6f3dc6e5d331a626dd4faaec2065339cf1142e64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T143715756210Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fbd4393d0e26dcc31d5b2da7f28dbbb9a2d36c2db918d86279474478baf2efc +size 6599185 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T143717072483Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T143717072483Z.json new file mode 100644 index 0000000000000000000000000000000000000000..14cf4bf6705887f157fb5243e739cbfedf30ae34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T143717072483Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441066496, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115654668288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115654668288, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115654668288, + "device": 0, + "inactive_split_bytes_current": 7269583872, + "inactive_split_bytes_peak": 14295266304, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:17.072079Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441066496, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115654668288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115654668288, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115654668288, + "device": 0, + "inactive_split_bytes_current": 7099714560, + "inactive_split_bytes_peak": 14295266304, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:15.881868Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7269583872, + "inactive_split_bytes_current_delta": 169869312, + "inactive_split_bytes_peak": 14295266304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118432464896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 423, + "segment_count_current_delta": 0, + "segment_count_peak": 423, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T143717072483Z.pickle", + "timestamp_utc": "2026-08-01T14:37:17.180570Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T143717072483Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T143717072483Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5d170a9a234e493bd379ddeb77896a79ed439cde --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T143717072483Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fbe8f8647bca069b9a702bcc6c1cd7092dd7433e0c50ef27811c4f280476aa6 +size 6029344 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T143718394252Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T143718394252Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8dfea4cf6e61c7e82807b1c8eb3e4750e67d1799 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T143718394252Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441066496, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115654668288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115654668288, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115654668288, + "device": 0, + "inactive_split_bytes_current": 8626441216, + "inactive_split_bytes_peak": 14295266304, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:18.393852Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441066496, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115654668288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115654668288, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115654668288, + "device": 0, + "inactive_split_bytes_current": 7269583872, + "inactive_split_bytes_peak": 14295266304, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:17.195961Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8626441216, + "inactive_split_bytes_current_delta": 1356857344, + "inactive_split_bytes_peak": 14295266304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118432464896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 423, + "segment_count_current_delta": 0, + "segment_count_peak": 423, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T143718394252Z.pickle", + "timestamp_utc": "2026-08-01T14:37:18.499253Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T143718394252Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T143718394252Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c49c4b79432aba7deab5903dcac6d78ad990e29f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T143718394252Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb9095130dcd2999e3a6e29f5b0d635880da7d25bbcfc6fbeba57c5ec3f0ced3 +size 6080208 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T143719872679Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T143719872679Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f8024c9dd3a9a6415c71608fe73a115df17f746c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T143719872679Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441066496, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115654668288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115654668288, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115654668288, + "device": 0, + "inactive_split_bytes_current": 7800163328, + "inactive_split_bytes_peak": 14295266304, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:19.872191Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441066496, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115654668288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115654668288, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115654668288, + "device": 0, + "inactive_split_bytes_current": 8626441216, + "inactive_split_bytes_peak": 14295266304, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:18.514706Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7800163328, + "inactive_split_bytes_current_delta": -826277888, + "inactive_split_bytes_peak": 14295266304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118432464896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 423, + "segment_count_current_delta": 0, + "segment_count_peak": 423, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T143719872679Z.pickle", + "timestamp_utc": "2026-08-01T14:37:19.979637Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T143719872679Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T143719872679Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ab8591c4ebbbe7c37b4a0e6c669445b7ce81640 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T143719872679Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0d4809313d14570cd000e12d6548d35011020da51761371e34eb744bd0921c4 +size 6029533 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T143721183081Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T143721183081Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c295fa107e8cf5b43f2a3fa0fe54a8d7e71a9548 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T143721183081Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441066496, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115654668288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115654668288, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115654668288, + "device": 0, + "inactive_split_bytes_current": 7624002560, + "inactive_split_bytes_peak": 14295266304, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:21.182679Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441066496, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115654668288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115654668288, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115654668288, + "device": 0, + "inactive_split_bytes_current": 7800163328, + "inactive_split_bytes_peak": 14295266304, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:19.994912Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7624002560, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 14295266304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118432464896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 423, + "segment_count_current_delta": 0, + "segment_count_peak": 423, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T143721183081Z.pickle", + "timestamp_utc": "2026-08-01T14:37:21.288973Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T143721183081Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T143721183081Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..946ae5e068452165028747a67fb073a12de8e179 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T143721183081Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:542b4b6a08654f44f8fb692cbcb5c800781717c8aba40e229a383b66a90447e7 +size 6079861 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T143819525248Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T143819525248Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3055fd602fc9526f5f007cf7bd6b739d0847ca5e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T143819525248Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166777344, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 115654668288, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166777344, + "active_bytes_peak": 115654668288, + "allocated_bytes_current": 61166777344, + "allocated_bytes_peak": 115654668288, + "device": 0, + "inactive_split_bytes_current": 8559332352, + "inactive_split_bytes_peak": 14295266304, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:38:19.524784Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166777344, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 115654668288, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441066496, + "active_bytes_peak": 115654668288, + "allocated_bytes_current": 72441066496, + "allocated_bytes_peak": 115654668288, + "device": 0, + "inactive_split_bytes_current": 7624002560, + "inactive_split_bytes_peak": 14295266304, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_peak": 118432464896, + "segment_count_current": 423, + "segment_count_peak": 423, + "timestamp_utc": "2026-08-01T14:37:21.304308Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8559332352, + "inactive_split_bytes_current_delta": 935329792, + "inactive_split_bytes_peak": 14295266304, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 118432464896, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118432464896, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 423, + "segment_count_current_delta": 0, + "segment_count_peak": 423, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T143819525248Z.pickle", + "timestamp_utc": "2026-08-01T14:38:19.727232Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T143819525248Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T143819525248Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7768ba2b7f2463e49353f647e3ff73dd039d6614 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T143819525248Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbd711d34ddb75a9666d2b756f9af5a1aede195b8db12ca877d2ba4f2481f710 +size 9857512 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T143710165628Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T143710165628Z.json new file mode 100644 index 0000000000000000000000000000000000000000..853d1a05acec9ab550782281a24131a058b56000 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T143710165628Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 92010463232, + "active_bytes_peak_delta": 56544126976, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92010463232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 92010463232, + "device": 1, + "inactive_split_bytes_current": 6356241408, + "inactive_split_bytes_peak": 12344462848, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 95669977088, + "reserved_bytes_peak": 95669977088, + "segment_count_current": 369, + "segment_count_peak": 369, + "timestamp_utc": "2026-08-01T14:37:10.165026Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 92010463232, + "allocated_bytes_peak_delta": 56544126976, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:36:03.161291Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 6356241408, + "inactive_split_bytes_current_delta": 6223422464, + "inactive_split_bytes_peak": 12344462848, + "inactive_split_bytes_peak_delta": 12184076800, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 95669977088, + "reserved_bytes_current_delta": 56425971712, + "reserved_bytes_peak": 95669977088, + "reserved_bytes_peak_delta": 56425971712, + "segment_count_current": 369, + "segment_count_current_delta": 298, + "segment_count_peak": 369, + "segment_count_peak_delta": 298, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T143710165628Z.pickle", + "timestamp_utc": "2026-08-01T14:37:10.280085Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T143710165628Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T143710165628Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f1c6106a50d6a70210a9cb19dc6c9e8143b2fa73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T143710165628Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a1aa53f111e779cb90b5852fc0fb19f4cb04789bd54a22dfc2e6c32f39f43a1 +size 5912535 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T143711514010Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T143711514010Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c7a7c259072059c59533ceb50e122b033007396 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T143711514010Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115655350272, + "active_bytes_peak_delta": 23644887040, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115655350272, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655350272, + "device": 1, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_peak": 15887131648, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 117822193664, + "reserved_bytes_peak": 117822193664, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:37:11.513571Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115655350272, + "allocated_bytes_peak_delta": 23644887040, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92010463232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 92010463232, + "device": 1, + "inactive_split_bytes_current": 6356241408, + "inactive_split_bytes_peak": 12344462848, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 95669977088, + "reserved_bytes_peak": 95669977088, + "segment_count_current": 369, + "segment_count_peak": 369, + "timestamp_utc": "2026-08-01T14:37:10.297073Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_current_delta": 1348468736, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 3542668800, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 117822193664, + "reserved_bytes_current_delta": 22152216576, + "reserved_bytes_peak": 117822193664, + "reserved_bytes_peak_delta": 22152216576, + "segment_count_current": 421, + "segment_count_current_delta": 52, + "segment_count_peak": 421, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T143711514010Z.pickle", + "timestamp_utc": "2026-08-01T14:37:11.626399Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T143711514010Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T143711514010Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..846eb854dcd77188c8e421fa9bf309e069ff925c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T143711514010Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92d769869ae455c7b97d46bf41e1ead24aa6c9e0998de3396cd23bed14e8469e +size 5942057 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T143712862152Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T143712862152Z.json new file mode 100644 index 0000000000000000000000000000000000000000..999622c526f90c22be760461bb4a6cd3e3b0de54 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T143712862152Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115660220416, + "active_bytes_peak_delta": 4870144, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115660220416, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115660220416, + "device": 1, + "inactive_split_bytes_current": 8295058432, + "inactive_split_bytes_peak": 15887131648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.861760Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115660220416, + "allocated_bytes_peak_delta": 4870144, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115655350272, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655350272, + "device": 1, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_peak": 15887131648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 117822193664, + "reserved_bytes_peak": 117822193664, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:37:11.638807Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8295058432, + "inactive_split_bytes_current_delta": 590348288, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 422, + "segment_count_current_delta": 1, + "segment_count_peak": 422, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T143712862152Z.pickle", + "timestamp_utc": "2026-08-01T14:37:12.961584Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T143712862152Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T143712862152Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fab24119c6a69e8a451e0f178ab98070e81773a9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T143712862152Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9ed33d27ac3a37ec409eb97d884ba78915e95778cea16ac017e016205e55f23 +size 5942074 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T143714194906Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T143714194906Z.json new file mode 100644 index 0000000000000000000000000000000000000000..75da1c856fb7fecb169342ba52aba24777dee061 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T143714194906Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 6774784, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_peak": 15887131648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.194499Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 6774784, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115660220416, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115660220416, + "device": 1, + "inactive_split_bytes_current": 8295058432, + "inactive_split_bytes_peak": 15887131648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.973441Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_current_delta": 116391936, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T143714194906Z.pickle", + "timestamp_utc": "2026-08-01T14:37:14.297785Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T143714194906Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T143714194906Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a0f1cfbd741deed2deb7d5632cdf2dd2c1ba29f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T143714194906Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f951a9f8de7c05dab3a47280037e8cdf4bde2111b5c0aeb1bec85db874dd8b7 +size 6605757 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T143715756177Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T143715756177Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f650abe268a9d59558b1922707919729af35941d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T143715756177Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 9470512128, + "inactive_split_bytes_peak": 15887131648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.755755Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_peak": 15887131648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.309534Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9470512128, + "inactive_split_bytes_current_delta": 1059061760, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T143715756177Z.pickle", + "timestamp_utc": "2026-08-01T14:37:15.865415Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T143715756177Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T143715756177Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e1e09bf4d7f2d9733275d794c90e771b706084c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T143715756177Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cfa8ce4a09e47c2afbfc2cf6cbd2c21dbda9783ca1dbd6fc7a2cf49c00719e2 +size 6605833 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T143717072464Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T143717072464Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a659ec1520fb2c0317a0ed73617217dc59e53fdb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T143717072464Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 8646331392, + "inactive_split_bytes_peak": 16681066496, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.072028Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 9470512128, + "inactive_split_bytes_peak": 15887131648, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.877549Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8646331392, + "inactive_split_bytes_current_delta": -824180736, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 793934848, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T143717072464Z.pickle", + "timestamp_utc": "2026-08-01T14:37:17.183148Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T143717072464Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T143717072464Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9aa2cd2fc06f90afc7060204245d778fea153c81 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T143717072464Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adfa52a8e75df84494e2985c3c902e7e03c430289d2bc5e96a6d1096ca0d81b7 +size 6605608 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T143718394182Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T143718394182Z.json new file mode 100644 index 0000000000000000000000000000000000000000..136fbb5c523917d783185292141d89f53f6f4d73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T143718394182Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_peak": 16681066496, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.393781Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 8646331392, + "inactive_split_bytes_peak": 16681066496, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.197616Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_current_delta": 236978176, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T143718394182Z.pickle", + "timestamp_utc": "2026-08-01T14:37:18.491084Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T143718394182Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T143718394182Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b550141988c3f0b00484dbea186da6c13faca74e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T143718394182Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3c4a7d126fcf60893deabb40640d24f352125ee5cf673c322dd1a9ecb14a7ec +size 6605621 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T143719872566Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T143719872566Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a49e982be4f25485a41baf7e4e86fe767777742a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T143719872566Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 7943785472, + "inactive_split_bytes_peak": 16681066496, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:19.872143Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_peak": 16681066496, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.502417Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7943785472, + "inactive_split_bytes_current_delta": -939524096, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T143719872566Z.pickle", + "timestamp_utc": "2026-08-01T14:37:19.976923Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T143719872566Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T143719872566Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ce224e64380612be1e6ee55a4a114618ea67fdd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T143719872566Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf3268e58da68e541e42742d74596081dccf04f1069f27b46db1863fc0951f62 +size 6605543 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T143721183026Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T143721183026Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36031c5c192288909110c376a23553fb8de85b7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T143721183026Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 9471560704, + "inactive_split_bytes_peak": 16681066496, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.182622Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 7943785472, + "inactive_split_bytes_peak": 16681066496, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:19.988887Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9471560704, + "inactive_split_bytes_current_delta": 1527775232, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T143721183026Z.pickle", + "timestamp_utc": "2026-08-01T14:37:21.280157Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T143721183026Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T143721183026Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..391364bd09ee201582a8971bdcf917e15f077e84 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T143721183026Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:378ed0ad08f81b8744b98e47361a0ff78734704f579c7d4d09fcea2d217cd984 +size 6605602 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T143819525262Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T143819525262Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a4aabf64963b8fedcd5495bfb500548ba4afd493 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T143819525262Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11275337728, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 10345024512, + "inactive_split_bytes_peak": 16681066496, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:38:19.524789Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11275337728, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115666995200, + "device": 1, + "inactive_split_bytes_current": 9471560704, + "inactive_split_bytes_peak": 16681066496, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.291583Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10345024512, + "inactive_split_bytes_current_delta": 873463808, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T143819525262Z.pickle", + "timestamp_utc": "2026-08-01T14:38:19.721678Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T143819525262Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T143819525262Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6e2b55294d63a52ff2ac2b24eef59542559470c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T143819525262Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7263ec212b1b9ed47729e7d998c19dbd2ff809269913fa08d7d6b609bff79b4 +size 9856728 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T143710165786Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T143710165786Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7accab2012dc1a7c900b3f4d4cef433fbedbac7a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T143710165786Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 92010463232, + "active_bytes_peak_delta": 56544126976, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92010463232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 92010463232, + "device": 2, + "inactive_split_bytes_current": 6358338560, + "inactive_split_bytes_peak": 12344462848, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 95669977088, + "reserved_bytes_peak": 95669977088, + "segment_count_current": 369, + "segment_count_peak": 369, + "timestamp_utc": "2026-08-01T14:37:10.165166Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 92010463232, + "allocated_bytes_peak_delta": 56544126976, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:36:03.160761Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6358338560, + "inactive_split_bytes_current_delta": 6225519616, + "inactive_split_bytes_peak": 12344462848, + "inactive_split_bytes_peak_delta": 12184076800, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 95669977088, + "reserved_bytes_current_delta": 56425971712, + "reserved_bytes_peak": 95669977088, + "reserved_bytes_peak_delta": 56425971712, + "segment_count_current": 369, + "segment_count_current_delta": 298, + "segment_count_peak": 369, + "segment_count_peak_delta": 298, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T143710165786Z.pickle", + "timestamp_utc": "2026-08-01T14:37:10.285613Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T143710165786Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T143710165786Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..76b2e9132eba1fd7669548cce7e9c9ea1ad8c82f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T143710165786Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1922b0202d7210667015d75529b4a2b2ba9c74b9a7e4e93cc824e08ab42b58d +size 5912545 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T143711513995Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T143711513995Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f0b865bf57864ae7a575a2681bf6a5207a0b5bb5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T143711513995Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115655350272, + "active_bytes_peak_delta": 23644887040, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115655350272, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655350272, + "device": 2, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_peak": 15887131648, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 117822193664, + "reserved_bytes_peak": 117822193664, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:37:11.513582Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115655350272, + "allocated_bytes_peak_delta": 23644887040, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92010463232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 92010463232, + "device": 2, + "inactive_split_bytes_current": 6358338560, + "inactive_split_bytes_peak": 12344462848, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 95669977088, + "reserved_bytes_peak": 95669977088, + "segment_count_current": 369, + "segment_count_peak": 369, + "timestamp_utc": "2026-08-01T14:37:10.304116Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_current_delta": 1346371584, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 3542668800, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 117822193664, + "reserved_bytes_current_delta": 22152216576, + "reserved_bytes_peak": 117822193664, + "reserved_bytes_peak_delta": 22152216576, + "segment_count_current": 421, + "segment_count_current_delta": 52, + "segment_count_peak": 421, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T143711513995Z.pickle", + "timestamp_utc": "2026-08-01T14:37:11.629718Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T143711513995Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T143711513995Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..11ed7de6c8426d77a858b27553fda20896e2a28b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T143711513995Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39f7e3c6944e2ce636cad306af2f494777da58975b747f601e0a68940eb1f6f0 +size 5942038 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T143712862131Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T143712862131Z.json new file mode 100644 index 0000000000000000000000000000000000000000..901116c2ceb17e25dac0c2f8cfdf894995e6973a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T143712862131Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115660220416, + "active_bytes_peak_delta": 4870144, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115660220416, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115660220416, + "device": 2, + "inactive_split_bytes_current": 8297155584, + "inactive_split_bytes_peak": 15887131648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.861737Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115660220416, + "allocated_bytes_peak_delta": 4870144, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115655350272, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655350272, + "device": 2, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_peak": 15887131648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 117822193664, + "reserved_bytes_peak": 117822193664, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:37:11.643492Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8297155584, + "inactive_split_bytes_current_delta": 592445440, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 422, + "segment_count_current_delta": 1, + "segment_count_peak": 422, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T143712862131Z.pickle", + "timestamp_utc": "2026-08-01T14:37:12.970401Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T143712862131Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T143712862131Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7b23bd70fb85579ba575a448100aeeeb55df12c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T143712862131Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25c8e015778ad5c8c58e5afbf70ef0ce07a29878c97172fea7958409b0a4b943 +size 5942074 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T143714194897Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T143714194897Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ba4e4e0c5559d233787abf130196d8c104c12db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T143714194897Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 6774784, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_peak": 15887131648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.194494Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 6774784, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115660220416, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115660220416, + "device": 2, + "inactive_split_bytes_current": 8297155584, + "inactive_split_bytes_peak": 15887131648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.986313Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_current_delta": 114294784, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T143714194897Z.pickle", + "timestamp_utc": "2026-08-01T14:37:14.305511Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T143714194897Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T143714194897Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c86d5cca746fa6897e68e9142095d01162044c98 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T143714194897Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09ce93056fc5702b9f29053d7c91f3bba53d84445c2f5310a2c1bfcfbd9641c2 +size 6605758 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T143715756179Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T143715756179Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2517cbbc039c7d5790af11e95a6a4ee6da051de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T143715756179Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 9470512128, + "inactive_split_bytes_peak": 15887131648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.755755Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_peak": 15887131648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.319192Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9470512128, + "inactive_split_bytes_current_delta": 1059061760, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T143715756179Z.pickle", + "timestamp_utc": "2026-08-01T14:37:15.865153Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T143715756179Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T143715756179Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd9f03a5b856cf6f027f20f63fd86c8cca0fe7ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T143715756179Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d43b45db7c0fc481d792781415fcd6ea8ef137d663b7d4fab974279b6bddde0a +size 6605835 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T143717072486Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T143717072486Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80c3e1ca9d255b6aed077e58d9b11f208c2f930c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T143717072486Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 8646331392, + "inactive_split_bytes_peak": 16681066496, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.072050Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 9470512128, + "inactive_split_bytes_peak": 15887131648, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.878568Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8646331392, + "inactive_split_bytes_current_delta": -824180736, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 793934848, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T143717072486Z.pickle", + "timestamp_utc": "2026-08-01T14:37:17.178502Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T143717072486Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T143717072486Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0ef8e11a747b1cbcff8ef245e7172d29808e60b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T143717072486Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb2cf130fe41d65a00078f8da049a0be1ef1330fec6451abe08f32e721655c75 +size 6605614 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T143718394211Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T143718394211Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d336c16779fc2420ac5e76eedebdb86de3fba274 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T143718394211Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_peak": 16681066496, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.393803Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 8646331392, + "inactive_split_bytes_peak": 16681066496, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.191535Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_current_delta": 236978176, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T143718394211Z.pickle", + "timestamp_utc": "2026-08-01T14:37:18.495772Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T143718394211Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T143718394211Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bad8265581e88dc6e46614030717c09ce2fda1d0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T143718394211Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b34f516a267142dab7e086265a0b61f571c5ba92272bfabad2c17392f8dbab5d +size 6605621 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T143719872584Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T143719872584Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3fa47a7db5ccff556dd41427a06e42a0e57b4203 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T143719872584Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 7943785472, + "inactive_split_bytes_peak": 16681066496, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:19.872166Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_peak": 16681066496, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.508580Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 7943785472, + "inactive_split_bytes_current_delta": -939524096, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T143719872584Z.pickle", + "timestamp_utc": "2026-08-01T14:37:19.978478Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T143719872584Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T143719872584Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cca3c746dbccc4440e907d059a12e22afe4c783a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T143719872584Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cd5c3c25c66732e0e1907a9165632dc6783b789645eed2662948d9e8dca7aa3 +size 6605541 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T143721183017Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T143721183017Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a75c9896684aae807b03413984cbf45f0acb287f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T143721183017Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 9471560704, + "inactive_split_bytes_peak": 16681066496, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.182619Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 7943785472, + "inactive_split_bytes_peak": 16681066496, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:19.991598Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9471560704, + "inactive_split_bytes_current_delta": 1527775232, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T143721183017Z.pickle", + "timestamp_utc": "2026-08-01T14:37:21.290500Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T143721183017Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T143721183017Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d8609461a0cc3a2825a0c53a1b1e298adddbc6fa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T143721183017Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1467a88c01e375b8742330b4d1dcafb2946dccb183faa45e8db449e1d1d56cb +size 6605602 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T143819525259Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T143819525259Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b1d71128c03bea7a81f68934aa70c6f3ac449f0f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T143819525259Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11275337728, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 10345024512, + "inactive_split_bytes_peak": 16681066496, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:38:19.524781Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11275337728, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115666995200, + "device": 2, + "inactive_split_bytes_current": 9471560704, + "inactive_split_bytes_peak": 16681066496, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.305857Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10345024512, + "inactive_split_bytes_current_delta": 873463808, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T143819525259Z.pickle", + "timestamp_utc": "2026-08-01T14:38:19.733721Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T143819525259Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T143819525259Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..66c475592f451b1c14a9aeb80a34aea67055574b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T143819525259Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a22be2cc30cf3f8f4195cb4d1f5821ae933e9f0694e215584a7abb6f9a42bfd +size 9856728 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T143710165771Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T143710165771Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f2b7ffe896ba578a216220ab8a93c9b10c22a102 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T143710165771Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 92010463232, + "active_bytes_peak_delta": 56544126976, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92010463232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 92010463232, + "device": 3, + "inactive_split_bytes_current": 6356241408, + "inactive_split_bytes_peak": 12344462848, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 95669977088, + "reserved_bytes_peak": 95669977088, + "segment_count_current": 369, + "segment_count_peak": 369, + "timestamp_utc": "2026-08-01T14:37:10.165190Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 92010463232, + "allocated_bytes_peak_delta": 56544126976, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:36:03.161122Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6356241408, + "inactive_split_bytes_current_delta": 6223422464, + "inactive_split_bytes_peak": 12344462848, + "inactive_split_bytes_peak_delta": 12184076800, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 95669977088, + "reserved_bytes_current_delta": 56425971712, + "reserved_bytes_peak": 95669977088, + "reserved_bytes_peak_delta": 56425971712, + "segment_count_current": 369, + "segment_count_current_delta": 298, + "segment_count_peak": 369, + "segment_count_peak_delta": 298, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T143710165771Z.pickle", + "timestamp_utc": "2026-08-01T14:37:10.284954Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T143710165771Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T143710165771Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c2809955e9919b60fd0eb1b58f8fd74e302626b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T143710165771Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a47469d4c6a100acffd3ace44cd409f5ea087339ecd72845c64603e63b94298 +size 5912537 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T143711514010Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T143711514010Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ce3385845e46809fddc14ba13d43f1417a7ce42e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T143711514010Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115655350272, + "active_bytes_peak_delta": 23644887040, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115655350272, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655350272, + "device": 3, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_peak": 15887131648, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 117822193664, + "reserved_bytes_peak": 117822193664, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:37:11.513603Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115655350272, + "allocated_bytes_peak_delta": 23644887040, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92010463232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 92010463232, + "device": 3, + "inactive_split_bytes_current": 6356241408, + "inactive_split_bytes_peak": 12344462848, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 95669977088, + "reserved_bytes_peak": 95669977088, + "segment_count_current": 369, + "segment_count_peak": 369, + "timestamp_utc": "2026-08-01T14:37:10.303469Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_current_delta": 1348468736, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 3542668800, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 117822193664, + "reserved_bytes_current_delta": 22152216576, + "reserved_bytes_peak": 117822193664, + "reserved_bytes_peak_delta": 22152216576, + "segment_count_current": 421, + "segment_count_current_delta": 52, + "segment_count_peak": 421, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T143711514010Z.pickle", + "timestamp_utc": "2026-08-01T14:37:11.628680Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T143711514010Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T143711514010Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e22009f651a722b51bed3837e7235e767f81df61 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T143711514010Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c9e0342a251f3c947b0a33890a89fa112f716cbb70ebdfb9ae3b1f366b2e383 +size 5942060 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T143712862238Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T143712862238Z.json new file mode 100644 index 0000000000000000000000000000000000000000..78fae6ec45cefc0304a5936356236770eec15cb1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T143712862238Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115660220416, + "active_bytes_peak_delta": 4870144, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115660220416, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115660220416, + "device": 3, + "inactive_split_bytes_current": 8295058432, + "inactive_split_bytes_peak": 15887131648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.861786Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115660220416, + "allocated_bytes_peak_delta": 4870144, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115655350272, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655350272, + "device": 3, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_peak": 15887131648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 117822193664, + "reserved_bytes_peak": 117822193664, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:37:11.642745Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8295058432, + "inactive_split_bytes_current_delta": 590348288, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 422, + "segment_count_current_delta": 1, + "segment_count_peak": 422, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T143712862238Z.pickle", + "timestamp_utc": "2026-08-01T14:37:12.968362Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T143712862238Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T143712862238Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2fd1db991d078a4c43a1548b66809bfb57859059 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T143712862238Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fcfcd780cafaac1167d5c39b9d1e25e8b097d837e1a73fabe3510ec030e8bad +size 5942074 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T143714194942Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T143714194942Z.json new file mode 100644 index 0000000000000000000000000000000000000000..95dbb00ace758b1e85c970e59a100c6cdd0d168d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T143714194942Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 6774784, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_peak": 15887131648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.194528Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 6774784, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115660220416, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115660220416, + "device": 3, + "inactive_split_bytes_current": 8295058432, + "inactive_split_bytes_peak": 15887131648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.982100Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_current_delta": 116391936, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T143714194942Z.pickle", + "timestamp_utc": "2026-08-01T14:37:14.301412Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T143714194942Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T143714194942Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c5aca5376ff76db8680a0fa4c8289c19c75cae24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T143714194942Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ad7ba448f5056382978c955129d7f5e39303442918fbd28da0d5ede389590ad +size 6605726 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T143715756190Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T143715756190Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b17ffef05b68ff23b812f30188efc8a0ca38d05d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T143715756190Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 9472609280, + "inactive_split_bytes_peak": 15887131648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.755791Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_peak": 15887131648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.314813Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9472609280, + "inactive_split_bytes_current_delta": 1061158912, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T143715756190Z.pickle", + "timestamp_utc": "2026-08-01T14:37:15.863156Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T143715756190Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T143715756190Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a3400cf9ba0290317ba649262c70e21709adcb29 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T143715756190Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62b89033fa6ac907a286e5610256e4afc2fc41a97381bb48960cc51e3de6ca48 +size 6605871 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T143717072481Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T143717072481Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ea32786405bba234d84ec62f22e96b17ce1039ee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T143717072481Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 8648428544, + "inactive_split_bytes_peak": 16681066496, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.072056Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 9472609280, + "inactive_split_bytes_peak": 15887131648, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.876854Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8648428544, + "inactive_split_bytes_current_delta": -824180736, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 793934848, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T143717072481Z.pickle", + "timestamp_utc": "2026-08-01T14:37:17.180528Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T143717072481Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T143717072481Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cdab2c3800428b02e6e22443a27b9e1c563ba348 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T143717072481Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f13e97f386b1a43700984ee596ecfd98cf16f46124d3c5db0f1c01ef63d12e7 +size 6605618 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T143718394213Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T143718394213Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d9ccbf03441a69acf28805cea21af292f82df200 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T143718394213Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_peak": 16681066496, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.393813Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 8648428544, + "inactive_split_bytes_peak": 16681066496, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.193926Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T143718394213Z.pickle", + "timestamp_utc": "2026-08-01T14:37:18.496837Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T143718394213Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T143718394213Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7d01c9e96bcc53e7b127500d3d06db3d78262954 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T143718394213Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:916457a93f574c79f1560fa92e0a712e5c4b61abca75ae88f1a65663033b8068 +size 6605593 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T143719872593Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T143719872593Z.json new file mode 100644 index 0000000000000000000000000000000000000000..87a3a07763c26104dbb8f730e40c28f980515f11 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T143719872593Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 7945882624, + "inactive_split_bytes_peak": 16681066496, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:19.872191Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_peak": 16681066496, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.510007Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 7945882624, + "inactive_split_bytes_current_delta": -937426944, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T143719872593Z.pickle", + "timestamp_utc": "2026-08-01T14:37:19.984705Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T143719872593Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T143719872593Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7836a37cf3f2649cc7644c0db9e617d5f3151532 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T143719872593Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d36e327e0c52e2f909af5b813633831960bff57a6808c70416ea181af61fc9fd +size 6605577 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T143721183039Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T143721183039Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1c0c36f9b4ec1d7644f86098e6fd75d629fad3cd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T143721183039Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 9473657856, + "inactive_split_bytes_peak": 16681066496, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.182643Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 7945882624, + "inactive_split_bytes_peak": 16681066496, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:20.001184Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9473657856, + "inactive_split_bytes_current_delta": 1527775232, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T143721183039Z.pickle", + "timestamp_utc": "2026-08-01T14:37:21.285854Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T143721183039Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T143721183039Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..242d526070eba2d64de4b6ff33d369dbe9d7490c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T143721183039Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2a68f1e6da391e6a2804d573738c4e60b919b9838152a06e94408c0ecf2b34a +size 6605603 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T143819525366Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T143819525366Z.json new file mode 100644 index 0000000000000000000000000000000000000000..80af11fe6b55ba7f26341e0de1e9b3e706121932 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T143819525366Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11275337728, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 10345024512, + "inactive_split_bytes_peak": 16681066496, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:38:19.524804Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11275337728, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115666995200, + "device": 3, + "inactive_split_bytes_current": 9473657856, + "inactive_split_bytes_peak": 16681066496, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.298795Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10345024512, + "inactive_split_bytes_current_delta": 871366656, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T143819525366Z.pickle", + "timestamp_utc": "2026-08-01T14:38:19.726850Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T143819525366Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T143819525366Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..16c8994bd543ef22ab1f80d48339275c04ac5b75 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T143819525366Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c77a2a6f58dd06f9c05e4178e449675d0e8ff59c0fe94223c5d686b81cc9effd +size 9856691 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T143710165741Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T143710165741Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f71f38b6c3396aefce8591a7077f3aec0c1b3291 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T143710165741Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441408512, + "active_bytes_current_delta": 37209510912, + "active_bytes_peak": 92007739392, + "active_bytes_peak_delta": 56540960768, + "after": { + "active_bytes_current": 72441408512, + "active_bytes_peak": 92007739392, + "allocated_bytes_current": 72441408512, + "allocated_bytes_peak": 92007739392, + "device": 4, + "inactive_split_bytes_current": 5765583872, + "inactive_split_bytes_peak": 11845692928, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 94845796352, + "reserved_bytes_peak": 94845796352, + "segment_count_current": 366, + "segment_count_peak": 366, + "timestamp_utc": "2026-08-01T14:37:10.165141Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441408512, + "allocated_bytes_current_delta": 37209510912, + "allocated_bytes_peak": 92007739392, + "allocated_bytes_peak_delta": 56540960768, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:36:03.160553Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5765583872, + "inactive_split_bytes_current_delta": 5633207296, + "inactive_split_bytes_peak": 11845692928, + "inactive_split_bytes_peak_delta": 11692283392, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 94845796352, + "reserved_bytes_current_delta": 55601790976, + "reserved_bytes_peak": 94845796352, + "reserved_bytes_peak_delta": 55601790976, + "segment_count_current": 366, + "segment_count_current_delta": 295, + "segment_count_peak": 366, + "segment_count_peak_delta": 295, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T143710165741Z.pickle", + "timestamp_utc": "2026-08-01T14:37:10.284464Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T143710165741Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T143710165741Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0c0bd886d3500f9aca628502d6434ebc65597aea --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T143710165741Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47175953b93d37af467aa197a51a32515f6c46475d3dbf87bedd0e99eb38fe4b +size 5775191 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T143711514017Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T143711514017Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d521175b8dcf7affcfc1260005d6db86a273246f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T143711514017Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442457088, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115657078784, + "active_bytes_peak_delta": 23649339392, + "after": { + "active_bytes_current": 72442457088, + "active_bytes_peak": 115657078784, + "allocated_bytes_current": 72442457088, + "allocated_bytes_peak": 115657078784, + "device": 4, + "inactive_split_bytes_current": 6293017600, + "inactive_split_bytes_peak": 14468439040, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:11.513607Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442457088, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115657078784, + "allocated_bytes_peak_delta": 23649339392, + "before": { + "active_bytes_current": 72441408512, + "active_bytes_peak": 92007739392, + "allocated_bytes_current": 72441408512, + "allocated_bytes_peak": 92007739392, + "device": 4, + "inactive_split_bytes_current": 5765583872, + "inactive_split_bytes_peak": 11845692928, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 94845796352, + "reserved_bytes_peak": 94845796352, + "segment_count_current": 366, + "segment_count_peak": 366, + "timestamp_utc": "2026-08-01T14:37:10.303540Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6293017600, + "inactive_split_bytes_current_delta": 527433728, + "inactive_split_bytes_peak": 14468439040, + "inactive_split_bytes_peak_delta": 2622746112, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 118409396224, + "reserved_bytes_current_delta": 23563599872, + "reserved_bytes_peak": 118409396224, + "reserved_bytes_peak_delta": 23563599872, + "segment_count_current": 422, + "segment_count_current_delta": 56, + "segment_count_peak": 422, + "segment_count_peak_delta": 56, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T143711514017Z.pickle", + "timestamp_utc": "2026-08-01T14:37:11.630141Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T143711514017Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T143711514017Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..36ebf96ef5af89c59faf8e96fe34cf1ab79edeeb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T143711514017Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:148c4633f75f8e0ce20b9db4320d5d6915defef6218e4b11ed7cba5f2943b2a8 +size 5935641 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T143712862230Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T143712862230Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5159bc436a798405f572d10e5e635e2d3fdc2c7c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T143712862230Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442457088, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115657078784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442457088, + "active_bytes_peak": 115657078784, + "allocated_bytes_current": 72442457088, + "allocated_bytes_peak": 115657078784, + "device": 4, + "inactive_split_bytes_current": 6643241984, + "inactive_split_bytes_peak": 16354836480, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.861833Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442457088, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115657078784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442457088, + "active_bytes_peak": 115657078784, + "allocated_bytes_current": 72442457088, + "allocated_bytes_peak": 115657078784, + "device": 4, + "inactive_split_bytes_current": 6293017600, + "inactive_split_bytes_peak": 14468439040, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:11.644288Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6643241984, + "inactive_split_bytes_current_delta": 350224384, + "inactive_split_bytes_peak": 16354836480, + "inactive_split_bytes_peak_delta": 1886397440, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 118409396224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118409396224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T143712862230Z.pickle", + "timestamp_utc": "2026-08-01T14:37:12.974987Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T143712862230Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T143712862230Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fdf4458d7f585ba8fe34cacb3d292a7b1823d3ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T143712862230Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e3f4d1bcc4b2b1809687499ecf3bbe0a05129f4d8582473ec515968b6e033d2 +size 6060225 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T143714194938Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T143714194938Z.json new file mode 100644 index 0000000000000000000000000000000000000000..afe428ffa106a1c471c60fdf43b82fea75d62e79 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T143714194938Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441408512, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 115657078784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441408512, + "active_bytes_peak": 115657078784, + "allocated_bytes_current": 72441408512, + "allocated_bytes_peak": 115657078784, + "device": 4, + "inactive_split_bytes_current": 7116149760, + "inactive_split_bytes_peak": 16354836480, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.194542Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441408512, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 115657078784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442457088, + "active_bytes_peak": 115657078784, + "allocated_bytes_current": 72442457088, + "allocated_bytes_peak": 115657078784, + "device": 4, + "inactive_split_bytes_current": 6643241984, + "inactive_split_bytes_peak": 16354836480, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.991603Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7116149760, + "inactive_split_bytes_current_delta": 472907776, + "inactive_split_bytes_peak": 16354836480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 118409396224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118409396224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T143714194938Z.pickle", + "timestamp_utc": "2026-08-01T14:37:14.306011Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T143714194938Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T143714194938Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da4038ce72ea72eb6cd2ca069de6c4b40b69c46a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T143714194938Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ee7a08265a314659e540cda5980a81a014d354acfec9adca69eff3d1819ce22 +size 6613766 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T143715756199Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T143715756199Z.json new file mode 100644 index 0000000000000000000000000000000000000000..371a4e0a119e3a1bb79a2f76db15c4a2a9272d4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T143715756199Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441408512, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115657078784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441408512, + "active_bytes_peak": 115657078784, + "allocated_bytes_current": 72441408512, + "allocated_bytes_peak": 115657078784, + "device": 4, + "inactive_split_bytes_current": 6939988992, + "inactive_split_bytes_peak": 16354836480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.755790Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441408512, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115657078784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441408512, + "active_bytes_peak": 115657078784, + "allocated_bytes_current": 72441408512, + "allocated_bytes_peak": 115657078784, + "device": 4, + "inactive_split_bytes_current": 7116149760, + "inactive_split_bytes_peak": 16354836480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.320259Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6939988992, + "inactive_split_bytes_current_delta": -176160768, + "inactive_split_bytes_peak": 16354836480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 118409396224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118409396224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T143715756199Z.pickle", + "timestamp_utc": "2026-08-01T14:37:15.865977Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T143715756199Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T143715756199Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..546a088f5da299425c311934fc0c19808bf6fd8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T143715756199Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb1d5c5ad44cabc01c59320e562564cfb92bfae56447d32ea96366039d3c9932 +size 6613562 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T143717072507Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T143717072507Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a94392580e0b764885c03fd829636c2ec6dbf6fd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T143717072507Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72442457088, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115657078784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72442457088, + "active_bytes_peak": 115657078784, + "allocated_bytes_current": 72442457088, + "allocated_bytes_peak": 115657078784, + "device": 4, + "inactive_split_bytes_current": 8939623424, + "inactive_split_bytes_peak": 16354836480, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.072075Z", + "world_size": 8 + }, + "allocated_bytes_current": 72442457088, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115657078784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441408512, + "active_bytes_peak": 115657078784, + "allocated_bytes_current": 72441408512, + "allocated_bytes_peak": 115657078784, + "device": 4, + "inactive_split_bytes_current": 6939988992, + "inactive_split_bytes_peak": 16354836480, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.879816Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8939623424, + "inactive_split_bytes_current_delta": 1999634432, + "inactive_split_bytes_peak": 16354836480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 118409396224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118409396224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T143717072507Z.pickle", + "timestamp_utc": "2026-08-01T14:37:17.180471Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T143717072507Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T143717072507Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b1acbcda88d50c0747d8c5d0e1f7a53d10da930f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T143717072507Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:785bb08aa9a4a4fdb7feb8ab7266ae3660d23166fbe72135b10d2a09164f5098 +size 6613499 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T143718394226Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T143718394226Z.json new file mode 100644 index 0000000000000000000000000000000000000000..841f5662bc60159c6d5443d51f98520fb2028b12 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T143718394226Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441408512, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 115657078784, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441408512, + "active_bytes_peak": 115657078784, + "allocated_bytes_current": 72441408512, + "allocated_bytes_peak": 115657078784, + "device": 4, + "inactive_split_bytes_current": 8588350464, + "inactive_split_bytes_peak": 16354836480, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.393818Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441408512, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 115657078784, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72442457088, + "active_bytes_peak": 115657078784, + "allocated_bytes_current": 72442457088, + "allocated_bytes_peak": 115657078784, + "device": 4, + "inactive_split_bytes_current": 8939623424, + "inactive_split_bytes_peak": 16354836480, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.194001Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8588350464, + "inactive_split_bytes_current_delta": -351272960, + "inactive_split_bytes_peak": 16354836480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 118409396224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118409396224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T143718394226Z.pickle", + "timestamp_utc": "2026-08-01T14:37:18.499094Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T143718394226Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T143718394226Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0561d540d78efa6abb83e07286eebd1e93e6ec33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T143718394226Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd4087f765f1cf4b81dc3697f17a768280f9b3147c881e983d7e12708c9b2afd +size 6613474 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T143719872592Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T143719872592Z.json new file mode 100644 index 0000000000000000000000000000000000000000..24deffc76c6d86abe6cb58a0ccd72b78563a777e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T143719872592Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441408512, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115658061824, + "active_bytes_peak_delta": 983040, + "after": { + "active_bytes_current": 72441408512, + "active_bytes_peak": 115658061824, + "allocated_bytes_current": 72441408512, + "allocated_bytes_peak": 115658061824, + "device": 4, + "inactive_split_bytes_current": 6589764608, + "inactive_split_bytes_peak": 16354836480, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:19.872184Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441408512, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115658061824, + "allocated_bytes_peak_delta": 983040, + "before": { + "active_bytes_current": 72441408512, + "active_bytes_peak": 115657078784, + "allocated_bytes_current": 72441408512, + "allocated_bytes_peak": 115657078784, + "device": 4, + "inactive_split_bytes_current": 8588350464, + "inactive_split_bytes_peak": 16354836480, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.512477Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6589764608, + "inactive_split_bytes_current_delta": -1998585856, + "inactive_split_bytes_peak": 16354836480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 118409396224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118409396224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T143719872592Z.pickle", + "timestamp_utc": "2026-08-01T14:37:19.980181Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T143719872592Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T143719872592Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..386dfe9add4978ec28aa5efabd712ac1bdacea46 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T143719872592Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8061848f8be59ef358ebcc3354de1e98d0cdf39ae874e05ba2bd00471762019e +size 6613492 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T143721183087Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T143721183087Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9bccf98e28f036ad2d2386fcd989ae83ce8ef3f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T143721183087Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441408512, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115660224512, + "active_bytes_peak_delta": 2162688, + "after": { + "active_bytes_current": 72441408512, + "active_bytes_peak": 115660224512, + "allocated_bytes_current": 72441408512, + "allocated_bytes_peak": 115660224512, + "device": 4, + "inactive_split_bytes_current": 8351372288, + "inactive_split_bytes_peak": 16354836480, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.182674Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441408512, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115660224512, + "allocated_bytes_peak_delta": 2162688, + "before": { + "active_bytes_current": 72441408512, + "active_bytes_peak": 115658061824, + "allocated_bytes_current": 72441408512, + "allocated_bytes_peak": 115658061824, + "device": 4, + "inactive_split_bytes_current": 6589764608, + "inactive_split_bytes_peak": 16354836480, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:19.993626Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8351372288, + "inactive_split_bytes_current_delta": 1761607680, + "inactive_split_bytes_peak": 16354836480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 118409396224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118409396224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T143721183087Z.pickle", + "timestamp_utc": "2026-08-01T14:37:21.287649Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T143721183087Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T143721183087Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2aec3de6e7ac05d8d75dd675ccd32b8e018679bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T143721183087Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68b38309a4e03ddb751eae3a5fbb6880945f4feae8361733b982963744fdf0c3 +size 6613431 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T143819525261Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T143819525261Z.json new file mode 100644 index 0000000000000000000000000000000000000000..802d86f0eb0d8723ae970f8d5ae56b49c656b876 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T143819525261Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61167119360, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 115660224512, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61167119360, + "active_bytes_peak": 115660224512, + "allocated_bytes_current": 61167119360, + "allocated_bytes_peak": 115660224512, + "device": 4, + "inactive_split_bytes_current": 8932283392, + "inactive_split_bytes_peak": 16354836480, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:38:19.524787Z", + "world_size": 8 + }, + "allocated_bytes_current": 61167119360, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 115660224512, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441408512, + "active_bytes_peak": 115660224512, + "allocated_bytes_current": 72441408512, + "allocated_bytes_peak": 115660224512, + "device": 4, + "inactive_split_bytes_current": 8351372288, + "inactive_split_bytes_peak": 16354836480, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 118409396224, + "reserved_bytes_peak": 118409396224, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.300851Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8932283392, + "inactive_split_bytes_current_delta": 580911104, + "inactive_split_bytes_peak": 16354836480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 118409396224, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118409396224, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T143819525261Z.pickle", + "timestamp_utc": "2026-08-01T14:38:19.733516Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T143819525261Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T143819525261Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..21d712f0ce16769f469c179e7016919abfda6247 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T143819525261Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b0b1fe08980f5c73a423718c93a172969ed33220e77ea90105e14d195aab19c +size 10518286 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T143710165655Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T143710165655Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cf6ac5ef2345a57afaa0f013fe68fc83ef55e239 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T143710165655Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 92010463232, + "active_bytes_peak_delta": 56544126976, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92010463232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 92010463232, + "device": 5, + "inactive_split_bytes_current": 6356241408, + "inactive_split_bytes_peak": 12344462848, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 95669977088, + "reserved_bytes_peak": 95669977088, + "segment_count_current": 369, + "segment_count_peak": 369, + "timestamp_utc": "2026-08-01T14:37:10.165043Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 92010463232, + "allocated_bytes_peak_delta": 56544126976, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:36:03.160352Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6356241408, + "inactive_split_bytes_current_delta": 6223422464, + "inactive_split_bytes_peak": 12344462848, + "inactive_split_bytes_peak_delta": 12184076800, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 95669977088, + "reserved_bytes_current_delta": 56425971712, + "reserved_bytes_peak": 95669977088, + "reserved_bytes_peak_delta": 56425971712, + "segment_count_current": 369, + "segment_count_current_delta": 298, + "segment_count_peak": 369, + "segment_count_peak_delta": 298, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T143710165655Z.pickle", + "timestamp_utc": "2026-08-01T14:37:10.283557Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T143710165655Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T143710165655Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..75fb0fb35e1e698095f480bb24db7d52e7ca9acb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T143710165655Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28e914f9bb217f3dbe2bd1024bed1463f38c305ab3ff4f6a25c7cec122aac880 +size 5912537 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T143711514003Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T143711514003Z.json new file mode 100644 index 0000000000000000000000000000000000000000..28bb9329ff485da8b876edc599c1d5c063afcc99 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T143711514003Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115655350272, + "active_bytes_peak_delta": 23644887040, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115655350272, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655350272, + "device": 5, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_peak": 15887131648, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 117822193664, + "reserved_bytes_peak": 117822193664, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:37:11.513579Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115655350272, + "allocated_bytes_peak_delta": 23644887040, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92010463232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 92010463232, + "device": 5, + "inactive_split_bytes_current": 6356241408, + "inactive_split_bytes_peak": 12344462848, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 95669977088, + "reserved_bytes_peak": 95669977088, + "segment_count_current": 369, + "segment_count_peak": 369, + "timestamp_utc": "2026-08-01T14:37:10.304144Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_current_delta": 1348468736, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 3542668800, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 117822193664, + "reserved_bytes_current_delta": 22152216576, + "reserved_bytes_peak": 117822193664, + "reserved_bytes_peak_delta": 22152216576, + "segment_count_current": 421, + "segment_count_current_delta": 52, + "segment_count_peak": 421, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T143711514003Z.pickle", + "timestamp_utc": "2026-08-01T14:37:11.631256Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T143711514003Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T143711514003Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..40436b5bc21ae917430a22b9d636b22d6769f740 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T143711514003Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32041936c9fbfe0fecdcae199800b11894979b35ed91a3d32ae70ff20dcb8168 +size 5942059 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T143712862140Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T143712862140Z.json new file mode 100644 index 0000000000000000000000000000000000000000..328a52f35c84ee0f8e3299ca3dbf3fd679ba27f2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T143712862140Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115660220416, + "active_bytes_peak_delta": 4870144, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115660220416, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115660220416, + "device": 5, + "inactive_split_bytes_current": 8295058432, + "inactive_split_bytes_peak": 15887131648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.861748Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115660220416, + "allocated_bytes_peak_delta": 4870144, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115655350272, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655350272, + "device": 5, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_peak": 15887131648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 117822193664, + "reserved_bytes_peak": 117822193664, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:37:11.647190Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8295058432, + "inactive_split_bytes_current_delta": 590348288, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 422, + "segment_count_current_delta": 1, + "segment_count_peak": 422, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T143712862140Z.pickle", + "timestamp_utc": "2026-08-01T14:37:12.973398Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T143712862140Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T143712862140Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f2b708c9e7101978e13fc827e12cd896c2e9ca08 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T143712862140Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26082fa91dfe3723b8b813b1f28929d2a062d8fa2a779e3c255a71c49b0f2702 +size 5942074 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T143714194941Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T143714194941Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ab43b54dcd0a8a102a23445957712b43c365e072 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T143714194941Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 6774784, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_peak": 15887131648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.194550Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 6774784, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115660220416, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115660220416, + "device": 5, + "inactive_split_bytes_current": 8295058432, + "inactive_split_bytes_peak": 15887131648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.988932Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_current_delta": 116391936, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T143714194941Z.pickle", + "timestamp_utc": "2026-08-01T14:37:14.306094Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T143714194941Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T143714194941Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44862edadaa974af9f40b2340532bf1b5985dbb2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T143714194941Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ca170b96686eb271f262021e8f8368f731149a98efd566cbe86bfff7cd40e17 +size 6605760 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T143715756183Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T143715756183Z.json new file mode 100644 index 0000000000000000000000000000000000000000..46cfe2e5183b3f496a40da5c9220138905d7d077 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T143715756183Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 9470512128, + "inactive_split_bytes_peak": 15887131648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.755762Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_peak": 15887131648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.321549Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9470512128, + "inactive_split_bytes_current_delta": 1059061760, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T143715756183Z.pickle", + "timestamp_utc": "2026-08-01T14:37:15.868269Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T143715756183Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T143715756183Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..17375a7327a9bcfa126e1c5f0d92b92c1665cf68 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T143715756183Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:422db4cc93501dc49f72c5125111894ce06b5f45926e8fc2a00370bd6be5c335 +size 6605833 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T143717072434Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T143717072434Z.json new file mode 100644 index 0000000000000000000000000000000000000000..16fcc5c70994387b9ab241a5ce482a26c6cc225f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T143717072434Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 8646331392, + "inactive_split_bytes_peak": 16681066496, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.072023Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 9470512128, + "inactive_split_bytes_peak": 15887131648, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.883910Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8646331392, + "inactive_split_bytes_current_delta": -824180736, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 793934848, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T143717072434Z.pickle", + "timestamp_utc": "2026-08-01T14:37:17.182390Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T143717072434Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T143717072434Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ba86b490ada87ce819b428446971b4ff2d811f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T143717072434Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c177392ee04f3119967427ae8153dac2cfb94896df3865ece7e0ebc667037bd +size 6605617 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T143718394206Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T143718394206Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e506059996916b9daf479e7f4671d1b55d35463 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T143718394206Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_peak": 16681066496, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.393802Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 8646331392, + "inactive_split_bytes_peak": 16681066496, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.197795Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_current_delta": 236978176, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T143718394206Z.pickle", + "timestamp_utc": "2026-08-01T14:37:18.505367Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T143718394206Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T143718394206Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b30888728f77a0a074e494839f78f2a74ac11191 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T143718394206Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6fba7a1c380db2ddf10c02cc5160355e4e2732bef515a711d2cd988100597b5 +size 6605621 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T143719872556Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T143719872556Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5d65de1bb39608dc97212da9ab1b14982fb80f64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T143719872556Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 7943785472, + "inactive_split_bytes_peak": 16681066496, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:19.872142Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_peak": 16681066496, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.523046Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7943785472, + "inactive_split_bytes_current_delta": -939524096, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T143719872556Z.pickle", + "timestamp_utc": "2026-08-01T14:37:19.985253Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T143719872556Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T143719872556Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..603168c7169652aef906bb38ca5ebfe6f32aa06b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T143719872556Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81ea688719232c4dc6c3d9eab1f2a6a4daef4b0996ce9fd9932d5f5c7ef58b70 +size 6605543 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T143721183027Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T143721183027Z.json new file mode 100644 index 0000000000000000000000000000000000000000..31f309da38ecb43b723d4545a88201d60573b83f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T143721183027Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 9471560704, + "inactive_split_bytes_peak": 16681066496, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.182623Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 7943785472, + "inactive_split_bytes_peak": 16681066496, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:20.000841Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9471560704, + "inactive_split_bytes_current_delta": 1527775232, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T143721183027Z.pickle", + "timestamp_utc": "2026-08-01T14:37:21.290034Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T143721183027Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T143721183027Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..286d39deaaf31e0533199d9fc4c06a0cbca641c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T143721183027Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8331ef22a688a722237c10f57d0d816adf60e4011d7a12d8ed2b3a7bdf542784 +size 6605602 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T143819525271Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T143819525271Z.json new file mode 100644 index 0000000000000000000000000000000000000000..28c540c36e3721c19cd485a8e24379830b91ef02 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T143819525271Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11275337728, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 10345024512, + "inactive_split_bytes_peak": 16681066496, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:38:19.524797Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11275337728, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115666995200, + "device": 5, + "inactive_split_bytes_current": 9471560704, + "inactive_split_bytes_peak": 16681066496, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.305090Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10345024512, + "inactive_split_bytes_current_delta": 873463808, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T143819525271Z.pickle", + "timestamp_utc": "2026-08-01T14:38:19.727807Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T143819525271Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T143819525271Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e027b0b854200fa9a89ad0de344fba4fa2944bdd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T143819525271Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1251320b9eb92c1e763827818041b448a21efca1d7a25cf5cea4212a4fc18662 +size 9856728 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T143710165704Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T143710165704Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fcc87fe60dc50931200621d94ddb3e24daa24c88 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T143710165704Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 92010463232, + "active_bytes_peak_delta": 56544126976, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92010463232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 92010463232, + "device": 6, + "inactive_split_bytes_current": 6356241408, + "inactive_split_bytes_peak": 12344462848, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 95669977088, + "reserved_bytes_peak": 95669977088, + "segment_count_current": 369, + "segment_count_peak": 369, + "timestamp_utc": "2026-08-01T14:37:10.165107Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 92010463232, + "allocated_bytes_peak_delta": 56544126976, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:36:03.160495Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6356241408, + "inactive_split_bytes_current_delta": 6223422464, + "inactive_split_bytes_peak": 12344462848, + "inactive_split_bytes_peak_delta": 12184076800, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 95669977088, + "reserved_bytes_current_delta": 56425971712, + "reserved_bytes_peak": 95669977088, + "reserved_bytes_peak_delta": 56425971712, + "segment_count_current": 369, + "segment_count_current_delta": 298, + "segment_count_peak": 369, + "segment_count_peak_delta": 298, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T143710165704Z.pickle", + "timestamp_utc": "2026-08-01T14:37:10.283975Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T143710165704Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T143710165704Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ada2e8e3ca725d2cec5548866f7fe5e0bbbdc68 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T143710165704Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21f616f3bff71dbf57fa0c233ffcb188c54b6b5b8fd071a9ea425a8ccefd86e2 +size 5912540 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T143711514001Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T143711514001Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8e632327afb523d04f45667ef56ca39fd8864466 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T143711514001Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115655350272, + "active_bytes_peak_delta": 23644887040, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115655350272, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655350272, + "device": 6, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_peak": 15887131648, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 117822193664, + "reserved_bytes_peak": 117822193664, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:37:11.513578Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115655350272, + "allocated_bytes_peak_delta": 23644887040, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92010463232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 92010463232, + "device": 6, + "inactive_split_bytes_current": 6356241408, + "inactive_split_bytes_peak": 12344462848, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 95669977088, + "reserved_bytes_peak": 95669977088, + "segment_count_current": 369, + "segment_count_peak": 369, + "timestamp_utc": "2026-08-01T14:37:10.310360Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_current_delta": 1348468736, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 3542668800, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 117822193664, + "reserved_bytes_current_delta": 22152216576, + "reserved_bytes_peak": 117822193664, + "reserved_bytes_peak_delta": 22152216576, + "segment_count_current": 421, + "segment_count_current_delta": 52, + "segment_count_peak": 421, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T143711514001Z.pickle", + "timestamp_utc": "2026-08-01T14:37:11.629602Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T143711514001Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T143711514001Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..75057ed9d6447233b8b43f86f8058664a15494b4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T143711514001Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27b19a593a284bc4abb06c7d9660347ea6f1f99f9d4f36ef2ae9963d8847e6b4 +size 5942056 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T143712862135Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T143712862135Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ae6902ca1ce5a318cc8fec67caccdcf14661dcfb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T143712862135Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115660220416, + "active_bytes_peak_delta": 4870144, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115660220416, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115660220416, + "device": 6, + "inactive_split_bytes_current": 8295058432, + "inactive_split_bytes_peak": 15887131648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.861745Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115660220416, + "allocated_bytes_peak_delta": 4870144, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115655350272, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655350272, + "device": 6, + "inactive_split_bytes_current": 7704710144, + "inactive_split_bytes_peak": 15887131648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 117822193664, + "reserved_bytes_peak": 117822193664, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:37:11.643535Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8295058432, + "inactive_split_bytes_current_delta": 590348288, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 422, + "segment_count_current_delta": 1, + "segment_count_peak": 422, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T143712862135Z.pickle", + "timestamp_utc": "2026-08-01T14:37:12.968875Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T143712862135Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T143712862135Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d76f16147358794f2e210b32a8f572bd84d0c8b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T143712862135Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:626f4773ec5d1a3489438ee8bb9e1e1db49d8b8710263136f3d4fb12a2c6dc85 +size 5942071 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T143714194892Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T143714194892Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9594a889683b0c6040398ac13a2384ce580dd742 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T143714194892Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 6774784, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_peak": 15887131648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.194492Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 6774784, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115660220416, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115660220416, + "device": 6, + "inactive_split_bytes_current": 8295058432, + "inactive_split_bytes_peak": 15887131648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.982475Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_current_delta": 116391936, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T143714194892Z.pickle", + "timestamp_utc": "2026-08-01T14:37:14.300703Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T143714194892Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T143714194892Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..24e7f50e7110942134d5f194dd475c106276497c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T143714194892Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00512972f4a3d4d310ecb99ed8d75b65c63eb1509cd450739f46c7d46b75b8a4 +size 6605760 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T143715756180Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T143715756180Z.json new file mode 100644 index 0000000000000000000000000000000000000000..30a8bfa2d2543b6311945e2c7795bdd5e0e6afe4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T143715756180Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 9470512128, + "inactive_split_bytes_peak": 15887131648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.755763Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 8411450368, + "inactive_split_bytes_peak": 15887131648, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.314178Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9470512128, + "inactive_split_bytes_current_delta": 1059061760, + "inactive_split_bytes_peak": 15887131648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T143715756180Z.pickle", + "timestamp_utc": "2026-08-01T14:37:15.864770Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T143715756180Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T143715756180Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c1c7d7023aaa54aee856df643071b3b89dc8f4c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T143715756180Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbd29ce6287c93cb0d8e21503711cc5590cab1eb586115c8e42a6e91eaa93176 +size 6605837 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T143717072459Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T143717072459Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6bdac9a215b6003c67425d63aa33abc6327a5997 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T143717072459Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 8646331392, + "inactive_split_bytes_peak": 16681066496, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.072026Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 9470512128, + "inactive_split_bytes_peak": 15887131648, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.878622Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8646331392, + "inactive_split_bytes_current_delta": -824180736, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 793934848, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T143717072459Z.pickle", + "timestamp_utc": "2026-08-01T14:37:17.179700Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T143717072459Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T143717072459Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ca2160a23b5a7a9d3615ffcb99dcb86529867323 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T143717072459Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82628605f5e5d2ffab18fa3d9b3b1b85337197e1465b7bb745d60632f591e867 +size 6605617 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T143718394211Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T143718394211Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9cfe97068c582e6f24423f81d28d11016245f9be --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T143718394211Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_peak": 16681066496, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.393806Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 8646331392, + "inactive_split_bytes_peak": 16681066496, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.193277Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_current_delta": 236978176, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T143718394211Z.pickle", + "timestamp_utc": "2026-08-01T14:37:18.496997Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T143718394211Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T143718394211Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..828da06f92ae140b5dbd0fd087725a98cf20a13a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T143718394211Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a60d835f984f886d30bfeb07a693f4a362e4a7b4e84731e2715dcef6c2b460 +size 6605621 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T143719872593Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T143719872593Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a776f6ba27b9b16825d9d2632e03cc1281f3c869 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T143719872593Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 7943785472, + "inactive_split_bytes_peak": 16681066496, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:19.872173Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 8883309568, + "inactive_split_bytes_peak": 16681066496, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.510300Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7943785472, + "inactive_split_bytes_current_delta": -939524096, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T143719872593Z.pickle", + "timestamp_utc": "2026-08-01T14:37:19.975277Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T143719872593Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T143719872593Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a4042b39691aed8abd875d6039d038b40cf67da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T143719872593Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:942e2101c592ef53abe8db4ba5b3ec609487ed5335fc4c8358649a6de8f0e20b +size 6605543 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T143721183039Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T143721183039Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c2ccf04862a192b3122eeb80f998a6c10483fe6d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T143721183039Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099264, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 9471560704, + "inactive_split_bytes_peak": 16681066496, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.182637Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099264, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 7943785472, + "inactive_split_bytes_peak": 16681066496, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:19.988604Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9471560704, + "inactive_split_bytes_current_delta": 1527775232, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T143721183039Z.pickle", + "timestamp_utc": "2026-08-01T14:37:21.285378Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T143721183039Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T143721183039Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f55ba47aede609685295dc4a5eaac04f2d2ba72b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T143721183039Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55c1999074a10c928a2049816ca6826b036e481aab2c451e102387979a4cb346 +size 6605602 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T143819525327Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T143819525327Z.json new file mode 100644 index 0000000000000000000000000000000000000000..401baf8c743ad8f4cfdbd7e585979e6673751c68 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T143819525327Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11275337728, + "active_bytes_peak": 115666995200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 10345024512, + "inactive_split_bytes_peak": 16681066496, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:38:19.524781Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11275337728, + "allocated_bytes_peak": 115666995200, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099264, + "active_bytes_peak": 115666995200, + "allocated_bytes_current": 72441099264, + "allocated_bytes_peak": 115666995200, + "device": 6, + "inactive_split_bytes_current": 9471560704, + "inactive_split_bytes_peak": 16681066496, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.298393Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10345024512, + "inactive_split_bytes_current_delta": 873463808, + "inactive_split_bytes_peak": 16681066496, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T143819525327Z.pickle", + "timestamp_utc": "2026-08-01T14:38:19.734904Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T143819525327Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T143819525327Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fd668513c4d6fb2b1e36bee4330c03c22b4d6aee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T143819525327Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:986abdcab6b1ada51b7a1d6dc4c44e7b0c63876d404f8a7fcddc6358a4ae5418 +size 9856728 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T143710165717Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T143710165717Z.json new file mode 100644 index 0000000000000000000000000000000000000000..79bbac58bf4f3d8f8f57b10dcb5e0a6f2a513eb9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T143710165717Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 37208595968, + "active_bytes_peak": 92010463232, + "active_bytes_peak_delta": 56544126976, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 92010463232, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 92010463232, + "device": 7, + "inactive_split_bytes_current": 6356240896, + "inactive_split_bytes_peak": 12344462848, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 95669977088, + "reserved_bytes_peak": 95669977088, + "segment_count_current": 369, + "segment_count_peak": 369, + "timestamp_utc": "2026-08-01T14:37:10.165136Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 37208595968, + "allocated_bytes_peak": 92010463232, + "allocated_bytes_peak_delta": 56544126976, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:36:03.160777Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6356240896, + "inactive_split_bytes_current_delta": 6223421952, + "inactive_split_bytes_peak": 12344462848, + "inactive_split_bytes_peak_delta": 12184076800, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 95669977088, + "reserved_bytes_current_delta": 56425971712, + "reserved_bytes_peak": 95669977088, + "reserved_bytes_peak_delta": 56425971712, + "segment_count_current": 369, + "segment_count_current_delta": 298, + "segment_count_peak": 369, + "segment_count_peak_delta": 298, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T143710165717Z.pickle", + "timestamp_utc": "2026-08-01T14:37:10.282336Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T143710165717Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T143710165717Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..920e512c682637efb163da78cc12a80dd2a50f70 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T143710165717Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81a98a1196582a8c1579e72b0be84f26bdc0c3c3404814f8c2d8df6a9f66a558 +size 5914141 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T143711514062Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T143711514062Z.json new file mode 100644 index 0000000000000000000000000000000000000000..55db0cad89a85b7459a1e6d44e02857b06723f03 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T143711514062Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115655350784, + "active_bytes_peak_delta": 23644887552, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 115655350784, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115655350784, + "device": 7, + "inactive_split_bytes_current": 7704709632, + "inactive_split_bytes_peak": 15887131136, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 117822193664, + "reserved_bytes_peak": 117822193664, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:37:11.513633Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115655350784, + "allocated_bytes_peak_delta": 23644887552, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 92010463232, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 92010463232, + "device": 7, + "inactive_split_bytes_current": 6356240896, + "inactive_split_bytes_peak": 12344462848, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 95669977088, + "reserved_bytes_peak": 95669977088, + "segment_count_current": 369, + "segment_count_peak": 369, + "timestamp_utc": "2026-08-01T14:37:10.303364Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7704709632, + "inactive_split_bytes_current_delta": 1348468736, + "inactive_split_bytes_peak": 15887131136, + "inactive_split_bytes_peak_delta": 3542668288, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 117822193664, + "reserved_bytes_current_delta": 22152216576, + "reserved_bytes_peak": 117822193664, + "reserved_bytes_peak_delta": 22152216576, + "segment_count_current": 421, + "segment_count_current_delta": 52, + "segment_count_peak": 421, + "segment_count_peak_delta": 52, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T143711514062Z.pickle", + "timestamp_utc": "2026-08-01T14:37:11.629611Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T143711514062Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T143711514062Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..798e70d1981d3efa4b7dcff7ae8bb8def2e47900 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T143711514062Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e404ad770182a6fa21ceaeac2def371f570bd3e520a5f0bc0ebf50ca78254ad2 +size 5943815 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T143712862179Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T143712862179Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bb25375ee5b2eaae4f93ae7d750dd80e916fa5ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T143712862179Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099776, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115660220928, + "active_bytes_peak_delta": 4870144, + "after": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 115660220928, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 115660220928, + "device": 7, + "inactive_split_bytes_current": 8295057920, + "inactive_split_bytes_peak": 15887131136, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.861748Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099776, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115660220928, + "allocated_bytes_peak_delta": 4870144, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 115655350784, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115655350784, + "device": 7, + "inactive_split_bytes_current": 7704709632, + "inactive_split_bytes_peak": 15887131136, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 117822193664, + "reserved_bytes_peak": 117822193664, + "segment_count_current": 421, + "segment_count_peak": 421, + "timestamp_utc": "2026-08-01T14:37:11.645654Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8295057920, + "inactive_split_bytes_current_delta": 590348288, + "inactive_split_bytes_peak": 15887131136, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 422, + "segment_count_current_delta": 1, + "segment_count_peak": 422, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T143712862179Z.pickle", + "timestamp_utc": "2026-08-01T14:37:12.975281Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T143712862179Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T143712862179Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c1975892a2a9c9c456d2e341962753d06e4bc904 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T143712862179Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f4e36e2a5189174816d0ebca2186be40f72f67edf2969e1c22aa4c7125f8259 +size 5943819 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T143714194972Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T143714194972Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f74a729147f0205df554d4f70c97a17870a03115 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T143714194972Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 115666995712, + "active_bytes_peak_delta": 6774784, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 8411449856, + "inactive_split_bytes_peak": 15887131136, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.194553Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 115666995712, + "allocated_bytes_peak_delta": 6774784, + "before": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 115660220928, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 115660220928, + "device": 7, + "inactive_split_bytes_current": 8295057920, + "inactive_split_bytes_peak": 15887131136, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:12.991251Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8411449856, + "inactive_split_bytes_current_delta": 116391936, + "inactive_split_bytes_peak": 15887131136, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T143714194972Z.pickle", + "timestamp_utc": "2026-08-01T14:37:14.310186Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T143714194972Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T143714194972Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2ac5553efaf7f3e450155ef495a3560425c1bd0d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T143714194972Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66bf72664237b6f13df3b971eb4137a4a4408d641440759b615286ee16c1a971 +size 6607478 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T143715756217Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T143715756217Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a71a21877727ac0efdf7d87c41e716d592595193 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T143715756217Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 9470511616, + "inactive_split_bytes_peak": 15887131136, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.755797Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 8411449856, + "inactive_split_bytes_peak": 15887131136, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:14.328638Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9470511616, + "inactive_split_bytes_current_delta": 1059061760, + "inactive_split_bytes_peak": 15887131136, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T143715756217Z.pickle", + "timestamp_utc": "2026-08-01T14:37:15.866814Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T143715756217Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T143715756217Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a25781b3dd960962b133c810df01e3c85216bcc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T143715756217Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a150a38449f6b460076b5dd1dc5facedd222baa97393801ab7df39e7c075fda6 +size 6607593 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T143717072487Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T143717072487Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c2d74ade29e2d6b4abc64718448942a476ec4c63 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T143717072487Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 8646330880, + "inactive_split_bytes_peak": 16681065984, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.072075Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 9470511616, + "inactive_split_bytes_peak": 15887131136, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:15.882704Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8646330880, + "inactive_split_bytes_current_delta": -824180736, + "inactive_split_bytes_peak": 16681065984, + "inactive_split_bytes_peak_delta": 793934848, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T143717072487Z.pickle", + "timestamp_utc": "2026-08-01T14:37:17.180956Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T143717072487Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T143717072487Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ed739ef0dfc7b38f11f253adc77956e85347907 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T143717072487Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d8ff811f52272c49e78ce5d30087de8fe8fc7ca760cd36471d74fa1ecc6d55c +size 6607365 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T143718394250Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T143718394250Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b17e7d53f391b1dc5f8436c57a5db1b7704ed5cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T143718394250Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 8883309056, + "inactive_split_bytes_peak": 16681065984, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.393840Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 8646330880, + "inactive_split_bytes_peak": 16681065984, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:17.196483Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8883309056, + "inactive_split_bytes_current_delta": 236978176, + "inactive_split_bytes_peak": 16681065984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T143718394250Z.pickle", + "timestamp_utc": "2026-08-01T14:37:18.502613Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T143718394250Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T143718394250Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de072f5592167ed627282331c6c4d8908f535255 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T143718394250Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6485680597a00ce46e88d80afcfd6b46cf1db05094c3ffd71b63c1f2d2073111 +size 6607342 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T143719872605Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T143719872605Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ac2a0febace7ea9f9187b60437497ee5e64d905 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T143719872605Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115666995712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 7943784960, + "inactive_split_bytes_peak": 16681065984, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:19.872201Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115666995712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 8883309056, + "inactive_split_bytes_peak": 16681065984, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:18.521925Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7943784960, + "inactive_split_bytes_current_delta": -939524096, + "inactive_split_bytes_peak": 16681065984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T143719872605Z.pickle", + "timestamp_utc": "2026-08-01T14:37:19.980738Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T143719872605Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T143719872605Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bdce8c8b5a719775d3a0126ede33242aaa90b015 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T143719872605Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95bce6e4edddb1c3253caf39201971fd2dc975794eeca7281c76ee39cefcc900 +size 6607290 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T143721183125Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T143721183125Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c696c80f03525468a690f4caa1910220cc062af6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T143721183125Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099776, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 115666995712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 9471560192, + "inactive_split_bytes_peak": 16681065984, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.182707Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099776, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115666995712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 7943784960, + "inactive_split_bytes_peak": 16681065984, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:19.996166Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9471560192, + "inactive_split_bytes_current_delta": 1527775232, + "inactive_split_bytes_peak": 16681065984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T143721183125Z.pickle", + "timestamp_utc": "2026-08-01T14:37:21.290293Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T143721183125Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T143721183125Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..70991602764c679167c262bf4cf911c269c6cb0d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T143721183125Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7627c2bba019e46f831e0d6bd08a08d5dd2a1e4ad852a910078de0e875e50629 +size 6607357 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T143819525235Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T143819525235Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5ae05e62708a58c560b36e3e407be65b2f9d629e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T143819525235Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165762048, + "active_bytes_current_delta": -11275337728, + "active_bytes_peak": 115666995712, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165762048, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 61165762048, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 10345024000, + "inactive_split_bytes_peak": 16681065984, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:38:19.524785Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165762048, + "allocated_bytes_current_delta": -11275337728, + "allocated_bytes_peak": 115666995712, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 115666995712, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 115666995712, + "device": 7, + "inactive_split_bytes_current": 9471560192, + "inactive_split_bytes_peak": 16681065984, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 118646374400, + "reserved_bytes_peak": 118646374400, + "segment_count_current": 422, + "segment_count_peak": 422, + "timestamp_utc": "2026-08-01T14:37:21.305755Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10345024000, + "inactive_split_bytes_current_delta": 873463808, + "inactive_split_bytes_peak": 16681065984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 118646374400, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 118646374400, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 422, + "segment_count_current_delta": 0, + "segment_count_peak": 422, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T143819525235Z.pickle", + "timestamp_utc": "2026-08-01T14:38:19.725730Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T143819525235Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T143819525235Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..701467ff33857ca1188f2a1060c40365504e83b1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T143819525235Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6693415bae6d14bf317222c1df7fc250417a9ba7f1d6ed9997922605d3544f7 +size 9858451 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..93bfb2bb39a711374a0cca77d36e1a94a50a60a2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 4096 +capacity: 1.50 +expert_dispatch_overlap: on +expert_dispatch_recompute: off +global_batch_size: 32 +train_iters: 10 +master_port: 6046 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 231 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T144107546892Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T144107546892Z.json new file mode 100644 index 0000000000000000000000000000000000000000..da259acd09bffb6b45437792525ff74c85a49ab9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T144107546892Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 31622601728, + "active_bytes_peak": 92114042880, + "active_bytes_peak_delta": 56647706624, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 92114042880, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 92114042880, + "device": 0, + "inactive_split_bytes_current": 5386537984, + "inactive_split_bytes_peak": 9635203584, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 98274639872, + "reserved_bytes_peak": 98274639872, + "segment_count_current": 432, + "segment_count_peak": 432, + "timestamp_utc": "2026-08-01T14:41:07.546497Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 31622601728, + "allocated_bytes_peak": 92114042880, + "allocated_bytes_peak_delta": 56647706624, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T14:39:58.590779Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5386537984, + "inactive_split_bytes_current_delta": 5251621888, + "inactive_split_bytes_peak": 9635203584, + "inactive_split_bytes_peak_delta": 9474571776, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 98274639872, + "reserved_bytes_current_delta": 59028537344, + "reserved_bytes_peak": 98274639872, + "reserved_bytes_peak_delta": 59028537344, + "segment_count_current": 432, + "segment_count_current_delta": 360, + "segment_count_peak": 432, + "segment_count_peak_delta": 360, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T144107546892Z.pickle", + "timestamp_utc": "2026-08-01T14:41:07.656404Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T144107546892Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T144107546892Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a37534503a63c9d60c44f683ae25c6b84431d105 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000000_20260801T144107546892Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76f2516dba3b67eb82fc4743c4048f0baf6c507a983315f3234b3db1a866fbd4 +size 5394786 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T144108759515Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T144108759515Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1b92330f82ab37e526990ba6a958830e66cfc1e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T144108759515Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115743522816, + "active_bytes_peak_delta": 23629479936, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115743522816, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115743522816, + "device": 0, + "inactive_split_bytes_current": 5170531328, + "inactive_split_bytes_peak": 10955383296, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:08.759105Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115743522816, + "allocated_bytes_peak_delta": 23629479936, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 92114042880, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 92114042880, + "device": 0, + "inactive_split_bytes_current": 5386537984, + "inactive_split_bytes_peak": 9635203584, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 98274639872, + "reserved_bytes_peak": 98274639872, + "segment_count_current": 432, + "segment_count_peak": 432, + "timestamp_utc": "2026-08-01T14:41:07.673744Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 5170531328, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 10955383296, + "inactive_split_bytes_peak_delta": 1320179712, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_current_delta": 21321744384, + "reserved_bytes_peak": 119596384256, + "reserved_bytes_peak_delta": 21321744384, + "segment_count_current": 489, + "segment_count_current_delta": 57, + "segment_count_peak": 489, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T144108759515Z.pickle", + "timestamp_utc": "2026-08-01T14:41:08.869313Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T144108759515Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T144108759515Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e4a25082362d874f3273cb86ea51eaf07f9ce863 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000001_20260801T144108759515Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9096eb10003d3e9af0f412515de1c9d8911efc20785790234959da53f549eb4 +size 5717023 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T144109965943Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T144109965943Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fade5f0b88f15917a64bf97b8ebdc16158eeaaf6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T144109965943Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115743674368, + "active_bytes_peak_delta": 151552, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115743674368, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115743674368, + "device": 0, + "inactive_split_bytes_current": 4931456000, + "inactive_split_bytes_peak": 10955383296, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:09.965545Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115743674368, + "allocated_bytes_peak_delta": 151552, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115743522816, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115743522816, + "device": 0, + "inactive_split_bytes_current": 5170531328, + "inactive_split_bytes_peak": 10955383296, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:08.882834Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4931456000, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 10955383296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119596384256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T144109965943Z.pickle", + "timestamp_utc": "2026-08-01T14:41:10.069863Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T144109965943Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T144109965943Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..495323deeba80221ed1e0ff5a5632620eadfb489 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000002_20260801T144109965943Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cbf3d10cd0ad4cf57eb7decbdf5f7899420a5ffb203e80beac3d9e73b2df0d3 +size 5614613 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T144111160368Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T144111160368Z.json new file mode 100644 index 0000000000000000000000000000000000000000..937955b0bec9b44ce36c54084558c4eeca36103a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T144111160368Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115743674368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115743674368, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115743674368, + "device": 0, + "inactive_split_bytes_current": 4929358848, + "inactive_split_bytes_peak": 11347369984, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:11.159974Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115743674368, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115743674368, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115743674368, + "device": 0, + "inactive_split_bytes_current": 4931456000, + "inactive_split_bytes_peak": 10955383296, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:10.083119Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4929358848, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11347369984, + "inactive_split_bytes_peak_delta": 391986688, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119596384256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T144111160368Z.pickle", + "timestamp_utc": "2026-08-01T14:41:11.262155Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T144111160368Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T144111160368Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5bd1633fafd21ded4880568af186b56d1e1d2d53 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000003_20260801T144111160368Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e9a488164d9666594f082bfbdecdbedc20a27ab3d426c08a5b9e25f2d884320 +size 5697454 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T144112594690Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T144112594690Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76c8184097f893f9d87eb6146a16078da7c79a56 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T144112594690Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115745771520, + "active_bytes_peak_delta": 2097152, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115745771520, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115745771520, + "device": 0, + "inactive_split_bytes_current": 4927261696, + "inactive_split_bytes_peak": 11347369984, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:12.594261Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115745771520, + "allocated_bytes_peak_delta": 2097152, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115743674368, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115743674368, + "device": 0, + "inactive_split_bytes_current": 4929358848, + "inactive_split_bytes_peak": 11347369984, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:11.275010Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4927261696, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 11347369984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119596384256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T144112594690Z.pickle", + "timestamp_utc": "2026-08-01T14:41:12.699419Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T144112594690Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T144112594690Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..092acd730f91f0de7fa55e29bc3152e8738eba34 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000004_20260801T144112594690Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abf42c6e3f56dfd728888aa23bd86a3704c713f7e592f7d1bf4e44e45bf74e25 +size 5613842 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T144113775158Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T144113775158Z.json new file mode 100644 index 0000000000000000000000000000000000000000..28179da8135b042e5edfbc8a7815177dfc2e2898 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T144113775158Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115745771520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115745771520, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115745771520, + "device": 0, + "inactive_split_bytes_current": 4927261696, + "inactive_split_bytes_peak": 11347369984, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:13.774765Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115745771520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115745771520, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115745771520, + "device": 0, + "inactive_split_bytes_current": 4927261696, + "inactive_split_bytes_peak": 11347369984, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:12.712710Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4927261696, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11347369984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119596384256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T144113775158Z.pickle", + "timestamp_utc": "2026-08-01T14:41:13.874875Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T144113775158Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T144113775158Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..60599715fc8c7f4225b755eb89d558b52ecb633f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000005_20260801T144113775158Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:916e24cc251a0042b1c642a0948b07aa55b2ea41d25e3fcd2b91137fdc8fe476 +size 5682469 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T144114947209Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T144114947209Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c4e1919606a5e4aad60c6f39dd7d6df498b1a86e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T144114947209Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115745771520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115745771520, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115745771520, + "device": 0, + "inactive_split_bytes_current": 4929358848, + "inactive_split_bytes_peak": 11347369984, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:14.946797Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115745771520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115745771520, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115745771520, + "device": 0, + "inactive_split_bytes_current": 4927261696, + "inactive_split_bytes_peak": 11347369984, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:13.887617Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4929358848, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11347369984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119596384256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T144114947209Z.pickle", + "timestamp_utc": "2026-08-01T14:41:15.051077Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T144114947209Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T144114947209Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..49985a2ec91bb20df952808c2de08d721e5f64b6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000006_20260801T144114947209Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aff688e72cb4c48d3f38ae5c281d7c600fc8c30defe1f52c606187d70c22cde3 +size 5618653 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T144116116429Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T144116116429Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b64f5e9960d32e7c8a3501c4076dbcbed462d380 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T144116116429Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115745771520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115745771520, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115745771520, + "device": 0, + "inactive_split_bytes_current": 4916775936, + "inactive_split_bytes_peak": 11347369984, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:16.115972Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115745771520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115745771520, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115745771520, + "device": 0, + "inactive_split_bytes_current": 4929358848, + "inactive_split_bytes_peak": 11347369984, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:15.063955Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4916775936, + "inactive_split_bytes_current_delta": -12582912, + "inactive_split_bytes_peak": 11347369984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119596384256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T144116116429Z.pickle", + "timestamp_utc": "2026-08-01T14:41:16.221382Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T144116116429Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T144116116429Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5f3c793f8012a6288d62f4915c048a474c21bd62 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000007_20260801T144116116429Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bdf8f9ee801b41e971545cc722f1752a209801bf5d10150f16681fe117fb534 +size 5688004 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T144117305769Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T144117305769Z.json new file mode 100644 index 0000000000000000000000000000000000000000..eddf0e8dac82226e6a116e586eca9d4a6d3e8711 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T144117305769Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66854056960, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115745771520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115745771520, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115745771520, + "device": 0, + "inactive_split_bytes_current": 4929358848, + "inactive_split_bytes_peak": 11347369984, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:17.305362Z", + "world_size": 8 + }, + "allocated_bytes_current": 66854056960, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115745771520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115745771520, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115745771520, + "device": 0, + "inactive_split_bytes_current": 4916775936, + "inactive_split_bytes_peak": 11347369984, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:16.234446Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4929358848, + "inactive_split_bytes_current_delta": 12582912, + "inactive_split_bytes_peak": 11347369984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119596384256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T144117305769Z.pickle", + "timestamp_utc": "2026-08-01T14:41:17.407102Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T144117305769Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T144117305769Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..66472066cdad9de59b04b2e8ee52a0321284c0f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000008_20260801T144117305769Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8876dcae8eb09ad4b0ae36c2115cd8d1870deb1af50b1afde1a449c7b76343c0 +size 5616896 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T144216459489Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T144216459489Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c75a7d87baf7903822b28991bfb5eb7f6a32eff9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T144216459489Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66803414016, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115745771520, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66803414016, + "active_bytes_peak": 115745771520, + "allocated_bytes_current": 66803414016, + "allocated_bytes_peak": 115745771520, + "device": 0, + "inactive_split_bytes_current": 4654943232, + "inactive_split_bytes_peak": 11347369984, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:42:16.459012Z", + "world_size": 8 + }, + "allocated_bytes_current": 66803414016, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115745771520, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66854056960, + "active_bytes_peak": 115745771520, + "allocated_bytes_current": 66854056960, + "allocated_bytes_peak": 115745771520, + "device": 0, + "inactive_split_bytes_current": 4929358848, + "inactive_split_bytes_peak": 11347369984, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_peak": 119596384256, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:17.422150Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 4654943232, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 11347369984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 119596384256, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119596384256, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T144216459489Z.pickle", + "timestamp_utc": "2026-08-01T14:42:16.664931Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T144216459489Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T144216459489Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..35bb4f4bb3620c3a7eedadec79f2d981357d550c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0000_iter000009_20260801T144216459489Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7fbd925c7679f7dc382e2d1828a116d1f2df7360a427df11038b570a9672a3c +size 10210490 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T144107546741Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T144107546741Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8288370fe9e6eb2a56e94d08f218607e342da679 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T144107546741Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 92108908544, + "active_bytes_peak_delta": 56642572288, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92108908544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92108908544, + "device": 1, + "inactive_split_bytes_current": 5424794624, + "inactive_split_bytes_peak": 10389252608, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 98742304768, + "reserved_bytes_peak": 98742304768, + "segment_count_current": 436, + "segment_count_peak": 436, + "timestamp_utc": "2026-08-01T14:41:07.546147Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 92108908544, + "allocated_bytes_peak_delta": 56642572288, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:39:58.591499Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5424794624, + "inactive_split_bytes_current_delta": 5291975680, + "inactive_split_bytes_peak": 10389252608, + "inactive_split_bytes_peak_delta": 10228866560, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 98742304768, + "reserved_bytes_current_delta": 59498299392, + "reserved_bytes_peak": 98742304768, + "reserved_bytes_peak_delta": 59498299392, + "segment_count_current": 436, + "segment_count_current_delta": 365, + "segment_count_peak": 436, + "segment_count_peak_delta": 365, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T144107546741Z.pickle", + "timestamp_utc": "2026-08-01T14:41:07.656113Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T144107546741Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T144107546741Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4535ef26854228e60c5b061f3e110dd3bbf32bfa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000000_20260801T144107546741Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f15a6f971df608ac11f6c2e399bf177d94687e2812d6fc5f8ed6b8285cc59a8 +size 5385465 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T144108759386Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T144108759386Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5c2c2c3f8f026669794c8cea4f4bcebf3dc3ce93 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T144108759386Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 23645974528, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5208787968, + "inactive_split_bytes_peak": 10389252608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 119355211776, + "reserved_bytes_peak": 119355211776, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:08.758983Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 23645974528, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92108908544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92108908544, + "device": 1, + "inactive_split_bytes_current": 5424794624, + "inactive_split_bytes_peak": 10389252608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 98742304768, + "reserved_bytes_peak": 98742304768, + "segment_count_current": 436, + "segment_count_peak": 436, + "timestamp_utc": "2026-08-01T14:41:07.673712Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5208787968, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 10389252608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 119355211776, + "reserved_bytes_current_delta": 20612907008, + "reserved_bytes_peak": 119355211776, + "reserved_bytes_peak_delta": 20612907008, + "segment_count_current": 489, + "segment_count_current_delta": 53, + "segment_count_peak": 489, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T144108759386Z.pickle", + "timestamp_utc": "2026-08-01T14:41:08.868403Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T144108759386Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T144108759386Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..63102a75ce6e1bce1dbddce8f3f90ef6fbfe0827 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000001_20260801T144108759386Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:966c5b523682760d30c0356247d27174fc4d7f2c4263a02647ba4995947bd3d1 +size 5613006 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T144109965818Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T144109965818Z.json new file mode 100644 index 0000000000000000000000000000000000000000..59b4017ebf8c162f66c575657a3ba923261fc448 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T144109965818Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5204593664, + "inactive_split_bytes_peak": 10473517056, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 119355211776, + "reserved_bytes_peak": 119355211776, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:09.965425Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5208787968, + "inactive_split_bytes_peak": 10389252608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 119355211776, + "reserved_bytes_peak": 119355211776, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:08.881716Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5204593664, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 10473517056, + "inactive_split_bytes_peak_delta": 84264448, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 119355211776, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119355211776, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T144109965818Z.pickle", + "timestamp_utc": "2026-08-01T14:41:10.069170Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T144109965818Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T144109965818Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8e9e512b0e31cf64f92b56c6455f243e575c3048 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000002_20260801T144109965818Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b11e451a2211ce4728e54497ee4bcb699e63f928e0bd1c5f583ee5c41a79b8e5 +size 5611250 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T144111160272Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T144111160272Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4f8dba2715f659349c59b65e0b1323ff8ca94638 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T144111160272Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5672258560, + "inactive_split_bytes_peak": 11050263040, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:11.159869Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5204593664, + "inactive_split_bytes_peak": 10473517056, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 119355211776, + "reserved_bytes_peak": 119355211776, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:10.082488Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5672258560, + "inactive_split_bytes_current_delta": 467664896, + "inactive_split_bytes_peak": 11050263040, + "inactive_split_bytes_peak_delta": 576745984, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 490, + "segment_count_current_delta": 1, + "segment_count_peak": 490, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T144111160272Z.pickle", + "timestamp_utc": "2026-08-01T14:41:11.260143Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T144111160272Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T144111160272Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9476dc8d7b1e1e16f1b9570dc0b2a01abacd07bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000003_20260801T144111160272Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f66b89e6cb34f8b6ef06f151ce63e9bc1678d2b1e58351e1b6e204db8858b45 +size 5612656 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T144112594495Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T144112594495Z.json new file mode 100644 index 0000000000000000000000000000000000000000..465974cbcffb9ac80fc83e9915f6ab0e35689f2e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T144112594495Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 11050263040, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:12.594087Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5672258560, + "inactive_split_bytes_peak": 11050263040, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:11.272985Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_current_delta": -471859200, + "inactive_split_bytes_peak": 11050263040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T144112594495Z.pickle", + "timestamp_utc": "2026-08-01T14:41:12.696356Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T144112594495Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T144112594495Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..840dd24da09107997540d7cffabd39eb6f9e2cbf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000004_20260801T144112594495Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1b27318cff8e86a37dd00af30182a5d21dc82240f48e58d6ac8b5dd8431f95e +size 5611241 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T144113775147Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T144113775147Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cdb2f13cc21cbb56413daab6032ea7c9fa9b8b93 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T144113775147Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5435280384, + "inactive_split_bytes_peak": 11050263040, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:13.774640Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 11050263040, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:12.709301Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5435280384, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 11050263040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T144113775147Z.pickle", + "timestamp_utc": "2026-08-01T14:41:13.873381Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T144113775147Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T144113775147Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..9b826dffb358057628af9695fa4ab20634e55cee --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000005_20260801T144113775147Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7a1f643859f437ded168cfe2edb733c3c4637b5114bbf830e6c87f985112190 +size 5612390 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T144114947072Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T144114947072Z.json new file mode 100644 index 0000000000000000000000000000000000000000..de540c76bc9dc1e5209887df82d70dab16ae3585 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T144114947072Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 11050263040, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:14.946674Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5435280384, + "inactive_split_bytes_peak": 11050263040, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:13.885938Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 11050263040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T144114947072Z.pickle", + "timestamp_utc": "2026-08-01T14:41:15.045870Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T144114947072Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T144114947072Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5a0deb5c1395f3e965481db475d688deafee9e4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000006_20260801T144114947072Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b8dac3bce6b8041e27de16ed39207f5e470205c4db6337b7e3956d1a821c036 +size 5612437 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T144116116279Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T144116116279Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a37c8a6b5239bcf150b641fae3eb347cc7cfd965 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T144116116279Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_peak": 11050263040, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:16.115841Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 11050263040, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:15.058665Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_current_delta": -12582912, + "inactive_split_bytes_peak": 11050263040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T144116116279Z.pickle", + "timestamp_utc": "2026-08-01T14:41:16.222652Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T144116116279Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T144116116279Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2fa921c3563e236eb3d4a8e1c8fd50332c0c4fb7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000007_20260801T144116116279Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5442faeaa1d25a1e38336d3d9199737c0187b110d1412391ef319e141f9cb127 +size 5612145 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T144117305620Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T144117305620Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6a685f8081d79de2d053246399368fcb8bf1c3f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T144117305620Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 11050263040, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:17.305232Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_peak": 11050263040, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:16.238158Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_current_delta": 12582912, + "inactive_split_bytes_peak": 11050263040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T144117305620Z.pickle", + "timestamp_utc": "2026-08-01T14:41:17.403466Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T144117305620Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T144117305620Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e6cebfb05a31dc09bf270d1edd973c7a54fc2b10 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000008_20260801T144117305620Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa0e5e28eebc541bce061c9970dbd34277e9629fd1abc52a8670cce2034c8069 +size 5611173 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T144216459504Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T144216459504Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d42fc2960da6b2d1b703d101a4e2fb4e022c0c9e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T144216459504Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 4928080896, + "inactive_split_bytes_peak": 11050263040, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:42:16.459021Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 1, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 11050263040, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:17.416059Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 4928080896, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 11050263040, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T144216459504Z.pickle", + "timestamp_utc": "2026-08-01T14:42:16.660972Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T144216459504Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T144216459504Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d565f25e78e8672862e92d2e0bd9da7f29c0ec7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0001_iter000009_20260801T144216459504Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8635daa5326967113a10c954a773b47c4f24f1e535c8f64851a47e33dedaffd5 +size 10209817 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T144107546720Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T144107546720Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9cee487b097234e46759adbd6aa65a85cd0ca851 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T144107546720Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 92108908544, + "active_bytes_peak_delta": 56642572288, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92108908544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92108908544, + "device": 2, + "inactive_split_bytes_current": 5424794624, + "inactive_split_bytes_peak": 10389252608, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 98742304768, + "reserved_bytes_peak": 98742304768, + "segment_count_current": 436, + "segment_count_peak": 436, + "timestamp_utc": "2026-08-01T14:41:07.546151Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 92108908544, + "allocated_bytes_peak_delta": 56642572288, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:39:58.590611Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5424794624, + "inactive_split_bytes_current_delta": 5291975680, + "inactive_split_bytes_peak": 10389252608, + "inactive_split_bytes_peak_delta": 10228866560, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 98742304768, + "reserved_bytes_current_delta": 59498299392, + "reserved_bytes_peak": 98742304768, + "reserved_bytes_peak_delta": 59498299392, + "segment_count_current": 436, + "segment_count_current_delta": 365, + "segment_count_peak": 436, + "segment_count_peak_delta": 365, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T144107546720Z.pickle", + "timestamp_utc": "2026-08-01T14:41:07.655109Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T144107546720Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T144107546720Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..69daa61655036f9c719be1e26d11ebace4142ee3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000000_20260801T144107546720Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6cee5141bd33daf32980d27c77aa8b66ae4de41541814948eea92db001495a2 +size 5385464 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T144108759386Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T144108759386Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7cfc9ff5ca6d1f4cdfe40145aa89391840476095 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T144108759386Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 23645974528, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5208787968, + "inactive_split_bytes_peak": 10389252608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 119355211776, + "reserved_bytes_peak": 119355211776, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:08.758967Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 23645974528, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92108908544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92108908544, + "device": 2, + "inactive_split_bytes_current": 5424794624, + "inactive_split_bytes_peak": 10389252608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 98742304768, + "reserved_bytes_peak": 98742304768, + "segment_count_current": 436, + "segment_count_peak": 436, + "timestamp_utc": "2026-08-01T14:41:07.672256Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5208787968, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 10389252608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 119355211776, + "reserved_bytes_current_delta": 20612907008, + "reserved_bytes_peak": 119355211776, + "reserved_bytes_peak_delta": 20612907008, + "segment_count_current": 489, + "segment_count_current_delta": 53, + "segment_count_peak": 489, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T144108759386Z.pickle", + "timestamp_utc": "2026-08-01T14:41:08.868790Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T144108759386Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T144108759386Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d229364afd027819cd6ee0c6caaa77566627ce42 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000001_20260801T144108759386Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ef3e1e5663a14c00c889c0932a2d175041bd3d38e1d5e861cd1092c6215f3bb +size 5612794 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T144109965826Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T144109965826Z.json new file mode 100644 index 0000000000000000000000000000000000000000..78817e577b72fa73da41f45aa393b54b8bff9bc3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T144109965826Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5204593664, + "inactive_split_bytes_peak": 10706300928, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 119355211776, + "reserved_bytes_peak": 119355211776, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:09.965417Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5208787968, + "inactive_split_bytes_peak": 10389252608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 119355211776, + "reserved_bytes_peak": 119355211776, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:08.882049Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5204593664, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 10706300928, + "inactive_split_bytes_peak_delta": 317048320, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 119355211776, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119355211776, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T144109965826Z.pickle", + "timestamp_utc": "2026-08-01T14:41:10.072630Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T144109965826Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T144109965826Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a7695fcf5e36f291a6c607eca5aec2e334afbae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000002_20260801T144109965826Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0dea18f148420d6e84a961a60cccca3025653fdbc36daaf141bf5de0b40d0a6 +size 5611422 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T144111160254Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T144111160254Z.json new file mode 100644 index 0000000000000000000000000000000000000000..920f85d2236a8e05ec5f4fed40dbf949b570b594 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T144111160254Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 10706300928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:11.159854Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5204593664, + "inactive_split_bytes_peak": 10706300928, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 119355211776, + "reserved_bytes_peak": 119355211776, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:10.088076Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 10706300928, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 490, + "segment_count_current_delta": 1, + "segment_count_peak": 490, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T144111160254Z.pickle", + "timestamp_utc": "2026-08-01T14:41:11.260480Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T144111160254Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T144111160254Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d9cdfd893b5970c544aefc4a555cff8bb84b6f7f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000003_20260801T144111160254Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1537459b5a5ba1bd17dc7d1809084f59ec1128e9a3badaa7bac579df67f4803 +size 5612657 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T144112594596Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T144112594596Z.json new file mode 100644 index 0000000000000000000000000000000000000000..1ae408b1abeba18167c714dacb65e08acea861e6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T144112594596Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 10772807680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:12.594101Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 10706300928, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:11.273208Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 10772807680, + "inactive_split_bytes_peak_delta": 66506752, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T144112594596Z.pickle", + "timestamp_utc": "2026-08-01T14:41:12.696486Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T144112594596Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T144112594596Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..559754a9707e5c12c41eecb4259ed5c7962af7b2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000004_20260801T144112594596Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1018ecc633dc37a7e5feb6b136c696727a146c4b0ab45c562c29b8e30ad783eb +size 5611265 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T144113775045Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T144113775045Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a24af73c0147f5e3de8992a01e1673fc997d162 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T144113775045Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5435280384, + "inactive_split_bytes_peak": 10772807680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:13.774637Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 10772807680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:12.709374Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5435280384, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 10772807680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T144113775045Z.pickle", + "timestamp_utc": "2026-08-01T14:41:13.872295Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T144113775045Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T144113775045Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea5461384a98f5638b7d46a3dbfceacbb4e4053f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000005_20260801T144113775045Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:318c1658436921de8fa7f3754b74102f4cf0e5586b3f2e25151c7a5aabbb1f19 +size 5612402 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T144114947084Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T144114947084Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ee05b2d811201ee0b005cfa8450f7681e745519 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T144114947084Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 10772807680, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:14.946679Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5435280384, + "inactive_split_bytes_peak": 10772807680, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:13.885139Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 10772807680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T144114947084Z.pickle", + "timestamp_utc": "2026-08-01T14:41:15.046200Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T144114947084Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T144114947084Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a08dcea11175858606ec95fbd26cac1ef48855b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000006_20260801T144114947084Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4cc3e53cf878f48fac928da9085e224f50e2509d1f644ffec95fbbb238f51c6 +size 5612449 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T144116116280Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T144116116280Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd0f14e4daf5beea39926510d93ffb025424af04 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T144116116280Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_peak": 10772807680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:16.115854Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 10772807680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:15.058746Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_current_delta": -12582912, + "inactive_split_bytes_peak": 10772807680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T144116116280Z.pickle", + "timestamp_utc": "2026-08-01T14:41:16.221058Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T144116116280Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T144116116280Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d0226e368c7fece427837372b7f6b3f6a81b0b0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000007_20260801T144116116280Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fea62c8ab61c66df6c1f42c4b3480db2463c5882d07f7d29190e344fee79f890 +size 5612225 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T144117305614Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T144117305614Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed83754a72ef3291f435e01f37e8f3b6e605dd35 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T144117305614Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 10772807680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:17.305224Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_peak": 10772807680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:16.236044Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_current_delta": 12582912, + "inactive_split_bytes_peak": 10772807680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T144117305614Z.pickle", + "timestamp_utc": "2026-08-01T14:41:17.401691Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T144117305614Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T144117305614Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1a7826c729afb16243d89c61f2754cff9e45f40b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000008_20260801T144117305614Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6588fa2f0c9285b184412736c62c2fff04714d557f486d40fac2047a45759256 +size 5611254 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T144216459577Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T144216459577Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2c5af9a174ffbc6804775099a96b9206e4c4cd2c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T144216459577Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 4928080896, + "inactive_split_bytes_peak": 10772807680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:42:16.459028Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 2, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 10772807680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:17.413914Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 4928080896, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 10772807680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T144216459577Z.pickle", + "timestamp_utc": "2026-08-01T14:42:16.657882Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T144216459577Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T144216459577Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..da496dbed2697dc810cc1fb0443d431732a291c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0002_iter000009_20260801T144216459577Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87bf15658c0857cfd7b729e3aa340ba8592d65882f887554386eae6458267816 +size 10209827 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T144107546687Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T144107546687Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a9337003e20c972442c90b1387b156acbda70953 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T144107546687Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 92108908544, + "active_bytes_peak_delta": 56642572288, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92108908544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92108908544, + "device": 3, + "inactive_split_bytes_current": 5659675648, + "inactive_split_bytes_peak": 10323716608, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 98507423744, + "reserved_bytes_peak": 98507423744, + "segment_count_current": 435, + "segment_count_peak": 435, + "timestamp_utc": "2026-08-01T14:41:07.546141Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 92108908544, + "allocated_bytes_peak_delta": 56642572288, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:39:58.590636Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5659675648, + "inactive_split_bytes_current_delta": 5526856704, + "inactive_split_bytes_peak": 10323716608, + "inactive_split_bytes_peak_delta": 10163330560, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 98507423744, + "reserved_bytes_current_delta": 59263418368, + "reserved_bytes_peak": 98507423744, + "reserved_bytes_peak_delta": 59263418368, + "segment_count_current": 435, + "segment_count_current_delta": 364, + "segment_count_peak": 435, + "segment_count_peak_delta": 364, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T144107546687Z.pickle", + "timestamp_utc": "2026-08-01T14:41:07.653950Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T144107546687Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T144107546687Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fa59f0a2b41629a457ee6f1c4519c4c3bac2f5ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000000_20260801T144107546687Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1330a3d86f73de4e67f491f1134002159e928c7628a353f999872847d1efea5 +size 5385368 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T144108759458Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T144108759458Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f0d719c754ca49d6c87fdf2f8615286a3d43a1c7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T144108759458Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115748067328, + "active_bytes_peak_delta": 23639158784, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115748067328, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115748067328, + "device": 3, + "inactive_split_bytes_current": 5443668992, + "inactive_split_bytes_peak": 10323716608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:08.759049Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115748067328, + "allocated_bytes_peak_delta": 23639158784, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92108908544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92108908544, + "device": 3, + "inactive_split_bytes_current": 5659675648, + "inactive_split_bytes_peak": 10323716608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 98507423744, + "reserved_bytes_peak": 98507423744, + "segment_count_current": 435, + "segment_count_peak": 435, + "timestamp_utc": "2026-08-01T14:41:07.670962Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5443668992, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 10323716608, + "inactive_split_bytes_peak_delta": 0, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119179051008, + "reserved_bytes_current_delta": 20671627264, + "reserved_bytes_peak": 119179051008, + "reserved_bytes_peak_delta": 20671627264, + "segment_count_current": 490, + "segment_count_current_delta": 55, + "segment_count_peak": 490, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T144108759458Z.pickle", + "timestamp_utc": "2026-08-01T14:41:08.871279Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T144108759458Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T144108759458Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5bbf7ebe6c19bb9d4cfb1ca752b9fba55572d3f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000001_20260801T144108759458Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fc0af9a26563bc5d2843b07a2d4afe6985c392ae440d3f0d804d6da8decb045 +size 5613374 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T144109965847Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T144109965847Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0bc2f4865a29744ca1233408a34250a5c47246d2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T144109965847Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115750950912, + "active_bytes_peak_delta": 2883584, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115750950912, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115750950912, + "device": 3, + "inactive_split_bytes_current": 5439474688, + "inactive_split_bytes_peak": 10724073984, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:09.965445Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115750950912, + "allocated_bytes_peak_delta": 2883584, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115748067328, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115748067328, + "device": 3, + "inactive_split_bytes_current": 5443668992, + "inactive_split_bytes_peak": 10323716608, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:08.884820Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5439474688, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 10724073984, + "inactive_split_bytes_peak_delta": 400357376, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119179051008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119179051008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T144109965847Z.pickle", + "timestamp_utc": "2026-08-01T14:41:10.070046Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T144109965847Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T144109965847Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..25b533055bfb81a4751aa6a51c2cdc55f30d75f6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000002_20260801T144109965847Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:810a3c02b9e1354ee706b7aa518827132fff441ea3b5d64f3bfe43b13f3e8fe0 +size 5612206 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T144111160291Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T144111160291Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0b88056a33c609d347b1da3a48e15a394f72e0c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T144111160291Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115750950912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115750950912, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115750950912, + "device": 3, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 10724073984, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:11.159890Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115750950912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115750950912, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115750950912, + "device": 3, + "inactive_split_bytes_current": 5439474688, + "inactive_split_bytes_peak": 10724073984, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:10.083315Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 10724073984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119179051008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119179051008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T144111160291Z.pickle", + "timestamp_utc": "2026-08-01T14:41:11.263466Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T144111160291Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T144111160291Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c796a0f459cfd1572df3ba4616ace2e6e680a51a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000003_20260801T144111160291Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9495fd7ca0a5bbbc0f0f5a8d15e95bb8b99eb5eb887c686ed1b6cb175e8ca369 +size 5613174 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T144112594520Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T144112594520Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2cc0654b96cf3867ab15ccd3af4402e0105be8bc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T144112594520Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115750950912, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115750950912, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115750950912, + "device": 3, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 10724073984, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:12.594114Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115750950912, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115750950912, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115750950912, + "device": 3, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 10724073984, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:11.276355Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 10724073984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119179051008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119179051008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T144112594520Z.pickle", + "timestamp_utc": "2026-08-01T14:41:12.697419Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T144112594520Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T144112594520Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8d1e5c8ec8823aa0dc35645d4737d1e30204e361 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000004_20260801T144112594520Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9da06559795f7a3a599103db588140683133e72c4f25b10b390ae90c484440be +size 5611785 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T144113775086Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T144113775086Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27d93fea35d64ad232c4f75d84ee6093126e40e7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T144113775086Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115751213056, + "active_bytes_peak_delta": 262144, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115751213056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115751213056, + "device": 3, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 10976817664, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:13.774688Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115751213056, + "allocated_bytes_peak_delta": 262144, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115750950912, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115750950912, + "device": 3, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 10724073984, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:12.710312Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10976817664, + "inactive_split_bytes_peak_delta": 252743680, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119179051008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119179051008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T144113775086Z.pickle", + "timestamp_utc": "2026-08-01T14:41:13.872906Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T144113775086Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T144113775086Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..79e7ab26eae301fd51b1267ac3552dd99b8c0050 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000005_20260801T144113775086Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb1e4b7c434b8b8899d90a08b362f1b92057a33a5ee83b5e0cb011492e2406e2 +size 5612710 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T144114947086Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T144114947086Z.json new file mode 100644 index 0000000000000000000000000000000000000000..248fc5733538409d3d0ebe4f7bafdd7b2da1e9f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T144114947086Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115751213056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115751213056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115751213056, + "device": 3, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 10978914816, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:14.946692Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115751213056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115751213056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115751213056, + "device": 3, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 10976817664, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:13.885523Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 10978914816, + "inactive_split_bytes_peak_delta": 2097152, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119179051008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119179051008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T144114947086Z.pickle", + "timestamp_utc": "2026-08-01T14:41:15.046999Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T144114947086Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T144114947086Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aa133df55d51decc0c8cc24b5b8233b30266d5a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000006_20260801T144114947086Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a79eb782f309ea8ec22521ea4a6beb6d16d45d1ffe3958a786860cbcf87b3389 +size 5612803 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T144116116281Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T144116116281Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e7ffd40387139f291bbd56cac261e87b71fe5071 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T144116116281Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115751213056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115751213056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115751213056, + "device": 3, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_peak": 10978914816, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:16.115866Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115751213056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115751213056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115751213056, + "device": 3, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 10978914816, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:15.059523Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_current_delta": -12582912, + "inactive_split_bytes_peak": 10978914816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 119179051008, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119179051008, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T144116116281Z.pickle", + "timestamp_utc": "2026-08-01T14:41:16.217122Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T144116116281Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T144116116281Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e18817eefe4bb1fbecd2b710e3a986add17177e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000007_20260801T144116116281Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e31771582a3873fd869fa1fe8ea2d8ab5c192abbd1aa3861036b851e7595f942 +size 5612580 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T144117305673Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T144117305673Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7089a63d56ea23d9e9c31a6b9337fa489ce7d044 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T144117305673Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115751213056, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115751213056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115751213056, + "device": 3, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 10978914816, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T14:41:17.305274Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115751213056, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115751213056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115751213056, + "device": 3, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_peak": 10978914816, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 119179051008, + "reserved_bytes_peak": 119179051008, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:16.229879Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_current_delta": 12582912, + "inactive_split_bytes_peak": 10978914816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120416370688, + "reserved_bytes_current_delta": 1237319680, + "reserved_bytes_peak": 120416370688, + "reserved_bytes_peak_delta": 1237319680, + "segment_count_current": 492, + "segment_count_current_delta": 2, + "segment_count_peak": 492, + "segment_count_peak_delta": 2, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T144117305673Z.pickle", + "timestamp_utc": "2026-08-01T14:41:17.408457Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T144117305673Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T144117305673Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ae5d6d59f305dd53b238422eaa6c16179abd2ce5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000008_20260801T144117305673Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66d8b9b3b6cb98851f0c6867e8c52737c36973d7d80a0f5ca808460b3a16eb88 +size 5611828 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T144216459490Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T144216459490Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9642fc498cee7133e6b33ce1a569d87fc9a09f64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T144216459490Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115751475200, + "active_bytes_peak_delta": 262144, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 115751475200, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 115751475200, + "device": 3, + "inactive_split_bytes_current": 4928080896, + "inactive_split_bytes_peak": 10978914816, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T14:42:16.459016Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115751475200, + "allocated_bytes_peak_delta": 262144, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115751213056, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115751213056, + "device": 3, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 10978914816, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 492, + "segment_count_peak": 492, + "timestamp_utc": "2026-08-01T14:41:17.421329Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 4928080896, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 10978914816, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 120416370688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120416370688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 492, + "segment_count_current_delta": 0, + "segment_count_peak": 492, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T144216459490Z.pickle", + "timestamp_utc": "2026-08-01T14:42:16.653721Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T144216459490Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T144216459490Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e55a3209dae9e4bf8ee860d586664d57578ef8b4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0003_iter000009_20260801T144216459490Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c4f6a9e14b98d0cdcb45e7d703a0584270b078941735b6a9c606468adbd90fe +size 9655628 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T144107546669Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T144107546669Z.json new file mode 100644 index 0000000000000000000000000000000000000000..620d7d0eb421e87591a0a304ff4ecaeee7e0515f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T144107546669Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855269376, + "active_bytes_current_delta": 31623371776, + "active_bytes_peak": 92108755456, + "active_bytes_peak_delta": 56641976832, + "after": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 92108755456, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 92108755456, + "device": 4, + "inactive_split_bytes_current": 5202873344, + "inactive_split_bytes_peak": 9481963008, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 98507423744, + "reserved_bytes_peak": 98507423744, + "segment_count_current": 434, + "segment_count_peak": 434, + "timestamp_utc": "2026-08-01T14:41:07.546259Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855269376, + "allocated_bytes_current_delta": 31623371776, + "allocated_bytes_peak": 92108755456, + "allocated_bytes_peak_delta": 56641976832, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:39:58.590996Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 5202873344, + "inactive_split_bytes_current_delta": 5070496768, + "inactive_split_bytes_peak": 9481963008, + "inactive_split_bytes_peak_delta": 9328553472, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 98507423744, + "reserved_bytes_current_delta": 59263418368, + "reserved_bytes_peak": 98507423744, + "reserved_bytes_peak_delta": 59263418368, + "segment_count_current": 434, + "segment_count_current_delta": 363, + "segment_count_peak": 434, + "segment_count_peak_delta": 363, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T144107546669Z.pickle", + "timestamp_utc": "2026-08-01T14:41:07.656974Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T144107546669Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T144107546669Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b00130e19513c19700732f476f69cb0ddedb87e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000000_20260801T144107546669Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec741e0f734ae46bff5c1f609a7ac07e8d6729ba014c1be60a2172f532434712 +size 5393628 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T144108759390Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T144108759390Z.json new file mode 100644 index 0000000000000000000000000000000000000000..618fa52b8ba7c127753c34811d979b5f057659ad --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T144108759390Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855269376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760566272, + "active_bytes_peak_delta": 23651810816, + "after": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4986866688, + "inactive_split_bytes_peak": 10264780800, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 119592189952, + "reserved_bytes_peak": 119592189952, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:08.758969Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855269376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760566272, + "allocated_bytes_peak_delta": 23651810816, + "before": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 92108755456, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 92108755456, + "device": 4, + "inactive_split_bytes_current": 5202873344, + "inactive_split_bytes_peak": 9481963008, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 98507423744, + "reserved_bytes_peak": 98507423744, + "segment_count_current": 434, + "segment_count_peak": 434, + "timestamp_utc": "2026-08-01T14:41:07.674381Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4986866688, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 10264780800, + "inactive_split_bytes_peak_delta": 782817792, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 119592189952, + "reserved_bytes_current_delta": 21084766208, + "reserved_bytes_peak": 119592189952, + "reserved_bytes_peak_delta": 21084766208, + "segment_count_current": 489, + "segment_count_current_delta": 55, + "segment_count_peak": 489, + "segment_count_peak_delta": 55, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T144108759390Z.pickle", + "timestamp_utc": "2026-08-01T14:41:08.874558Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T144108759390Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T144108759390Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..e320ff975c0034b8f66dc35d6a37907843f3ff8e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000001_20260801T144108759390Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25c2c5ddfca5a1e0849ab3f5e0669b49b075bdc43424e40dc80473caa392cf12 +size 5628885 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T144109965863Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T144109965863Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e44bdde604a18a23b3be6b7c38bf6b6317a1cf40 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T144109965863Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855269376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760566272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4982672384, + "inactive_split_bytes_peak": 10989573120, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:09.965451Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855269376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760566272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4986866688, + "inactive_split_bytes_peak": 10264780800, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 119592189952, + "reserved_bytes_peak": 119592189952, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:08.888025Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4982672384, + "inactive_split_bytes_current_delta": -4194304, + "inactive_split_bytes_peak": 10989573120, + "inactive_split_bytes_peak_delta": 724792320, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120416370688, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 120416370688, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 490, + "segment_count_current_delta": 1, + "segment_count_peak": 490, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T144109965863Z.pickle", + "timestamp_utc": "2026-08-01T14:41:10.071729Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T144109965863Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T144109965863Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a006646d80531834cdcfa6925a60ba1d37820dc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000002_20260801T144109965863Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee13377da515c82fef127a4002cd0db0d230f410c9c993ad36d13f75f07d2fae +size 5619386 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T144111160247Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T144111160247Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9e6ac570b0a0d34d6c64188548b8b746b3e5a975 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T144111160247Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855269376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760566272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4982672384, + "inactive_split_bytes_peak": 10989573120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:11.159845Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855269376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760566272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4982672384, + "inactive_split_bytes_peak": 10989573120, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:10.084937Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4982672384, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10989573120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120416370688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120416370688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T144111160247Z.pickle", + "timestamp_utc": "2026-08-01T14:41:11.270524Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T144111160247Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T144111160247Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6345541ebf20eed15c2802f998521d57436a4f4a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000003_20260801T144111160247Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6e7f7d796eb834fe42b1ee71b47b33479a18aa970ee6bcb042bb23c368c49df +size 5628241 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T144112594499Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T144112594499Z.json new file mode 100644 index 0000000000000000000000000000000000000000..59288d65f1f5a7d2768626fd7fb800d499ff4897 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T144112594499Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855269376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760566272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4980575232, + "inactive_split_bytes_peak": 10989573120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:12.594079Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855269376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760566272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4982672384, + "inactive_split_bytes_peak": 10989573120, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:11.286252Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4980575232, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 10989573120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120416370688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120416370688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T144112594499Z.pickle", + "timestamp_utc": "2026-08-01T14:41:12.704328Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T144112594499Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T144112594499Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b758797d6d98f6ae7d891c5f1162366e64229790 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000004_20260801T144112594499Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:841790e6abcff1394da4f5bd4e69503be6fc029b1e08b2ff5aba0fd0a83c1850 +size 5619119 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T144113775033Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T144113775033Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ddd70c4f15b4c680fe6edb93900f4a50007721ac --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T144113775033Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855269376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760566272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4980575232, + "inactive_split_bytes_peak": 10989573120, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:13.774624Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855269376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760566272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4980575232, + "inactive_split_bytes_peak": 10989573120, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:12.717683Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4980575232, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 10989573120, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120416370688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120416370688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T144113775033Z.pickle", + "timestamp_utc": "2026-08-01T14:41:13.876537Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T144113775033Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T144113775033Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2abe234cafabf88bdb20702cae615a78bac492f8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000005_20260801T144113775033Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:322aa5751bcbc9426fce644f5c6b93df963f00309ba2094085d65b17d32c3c63 +size 5622363 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T144114947059Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T144114947059Z.json new file mode 100644 index 0000000000000000000000000000000000000000..af1688a0671f0929d68bbc0602da6b0405650091 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T144114947059Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855269376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760566272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4982672384, + "inactive_split_bytes_peak": 11046196224, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:14.946656Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855269376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760566272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4980575232, + "inactive_split_bytes_peak": 10989573120, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:13.889303Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4982672384, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11046196224, + "inactive_split_bytes_peak_delta": 56623104, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120416370688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120416370688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T144114947059Z.pickle", + "timestamp_utc": "2026-08-01T14:41:15.046902Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T144114947059Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T144114947059Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb4a525ec097336c107103eebb9205956df66212 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000006_20260801T144114947059Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3564215afc320de7aaf3426fc87a76fcae2f0224cd240fe790405833b993e2ff +size 5620313 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T144116116254Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T144116116254Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed6540d3dcf0870b34c43251693a54d43e5a7235 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T144116116254Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855269376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760566272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4970089472, + "inactive_split_bytes_peak": 11046196224, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:16.115834Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855269376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760566272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4982672384, + "inactive_split_bytes_peak": 11046196224, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:15.059738Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4970089472, + "inactive_split_bytes_current_delta": -12582912, + "inactive_split_bytes_peak": 11046196224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120416370688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120416370688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T144116116254Z.pickle", + "timestamp_utc": "2026-08-01T14:41:16.218400Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T144116116254Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T144116116254Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..bd0ad02661978edd4fc0d4d1554c7aa2e508b648 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000007_20260801T144116116254Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00fcf889282dacc652a15b6e4536bcc489a3ea96a5cd65dbfbc5de7454ff6f02 +size 5626063 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T144117305659Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T144117305659Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4eee05416fc4ae17de046a60a987940ea67f4b51 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T144117305659Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66855269376, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115760566272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4982672384, + "inactive_split_bytes_peak": 11046196224, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:17.305244Z", + "world_size": 8 + }, + "allocated_bytes_current": 66855269376, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115760566272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4970089472, + "inactive_split_bytes_peak": 11046196224, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:16.231120Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4982672384, + "inactive_split_bytes_current_delta": 12582912, + "inactive_split_bytes_peak": 11046196224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120416370688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120416370688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T144117305659Z.pickle", + "timestamp_utc": "2026-08-01T14:41:17.406234Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T144117305659Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T144117305659Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..669ca869fdc3b773292b7848101b41e8bad35720 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000008_20260801T144117305659Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c77b4d0cfa02d7e50dd72451908d5c910a5f0a1598b1e6b85fa84a56539af2b9 +size 5619141 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T144216459491Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T144216459491Z.json new file mode 100644 index 0000000000000000000000000000000000000000..bd61f72e41e53968a15ecb23e22250a008e08a08 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T144216459491Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66804626432, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115760566272, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66804626432, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66804626432, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4708256768, + "inactive_split_bytes_peak": 11046196224, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:42:16.459023Z", + "world_size": 8 + }, + "allocated_bytes_current": 66804626432, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115760566272, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66855269376, + "active_bytes_peak": 115760566272, + "allocated_bytes_current": 66855269376, + "allocated_bytes_peak": 115760566272, + "device": 4, + "inactive_split_bytes_current": 4982672384, + "inactive_split_bytes_peak": 11046196224, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 120416370688, + "reserved_bytes_peak": 120416370688, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:17.418920Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 4708256768, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 11046196224, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 120416370688, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 120416370688, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 490, + "segment_count_current_delta": 0, + "segment_count_peak": 490, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T144216459491Z.pickle", + "timestamp_utc": "2026-08-01T14:42:16.664624Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T144216459491Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T144216459491Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5efe8acda1fd8f11e73fe41b77652b7d3d48569a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0004_iter000009_20260801T144216459491Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7403c2f67c1989882d171f9f21511f352cbd4365c3ea49ff095cd4fd766fa022 +size 10217801 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T144107546721Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T144107546721Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2ad1fcfb73073fa374286f8aa83b1e8112395d26 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T144107546721Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 92108908544, + "active_bytes_peak_delta": 56642572288, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92108908544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92108908544, + "device": 5, + "inactive_split_bytes_current": 5659675648, + "inactive_split_bytes_peak": 10389252608, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 98742304768, + "reserved_bytes_peak": 98742304768, + "segment_count_current": 436, + "segment_count_peak": 436, + "timestamp_utc": "2026-08-01T14:41:07.546123Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 92108908544, + "allocated_bytes_peak_delta": 56642572288, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:39:58.590750Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5659675648, + "inactive_split_bytes_current_delta": 5526856704, + "inactive_split_bytes_peak": 10389252608, + "inactive_split_bytes_peak_delta": 10228866560, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 98742304768, + "reserved_bytes_current_delta": 59498299392, + "reserved_bytes_peak": 98742304768, + "reserved_bytes_peak_delta": 59498299392, + "segment_count_current": 436, + "segment_count_current_delta": 365, + "segment_count_peak": 436, + "segment_count_peak_delta": 365, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T144107546721Z.pickle", + "timestamp_utc": "2026-08-01T14:41:07.656978Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T144107546721Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T144107546721Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2d9e5104782c939019bf8e8b57db571de4700a8f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000000_20260801T144107546721Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9366d44df43196693f1f3fac8cff4cab88600a670c66c6fd4d81fd1d987470ca +size 5385517 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T144108759391Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T144108759391Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cb80d3c7c40408019b26f9c06013af45445ef98a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T144108759391Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 23645974528, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5443668992, + "inactive_split_bytes_peak": 11252827648, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 119355211776, + "reserved_bytes_peak": 119355211776, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:08.758971Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 23645974528, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92108908544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92108908544, + "device": 5, + "inactive_split_bytes_current": 5659675648, + "inactive_split_bytes_peak": 10389252608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 98742304768, + "reserved_bytes_peak": 98742304768, + "segment_count_current": 436, + "segment_count_peak": 436, + "timestamp_utc": "2026-08-01T14:41:07.674370Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5443668992, + "inactive_split_bytes_current_delta": -216006656, + "inactive_split_bytes_peak": 11252827648, + "inactive_split_bytes_peak_delta": 863575040, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 119355211776, + "reserved_bytes_current_delta": 20612907008, + "reserved_bytes_peak": 119355211776, + "reserved_bytes_peak_delta": 20612907008, + "segment_count_current": 489, + "segment_count_current_delta": 53, + "segment_count_peak": 489, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T144108759391Z.pickle", + "timestamp_utc": "2026-08-01T14:41:08.868174Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T144108759391Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T144108759391Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3682d1eb2327ab11c3cac2ecb8617dfe38df26f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000001_20260801T144108759391Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1a7d2dcb3410a84f559bc7d1283c28ef25b97663d361c815bdad194fa44deae +size 5613594 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T144109965833Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T144109965833Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c1620f98d64359eac7ffe48e48daec29b35f2f38 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T144109965833Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5204593664, + "inactive_split_bytes_peak": 11252827648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 119355211776, + "reserved_bytes_peak": 119355211776, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:09.965413Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5443668992, + "inactive_split_bytes_peak": 11252827648, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 119355211776, + "reserved_bytes_peak": 119355211776, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:08.881544Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5204593664, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 11252827648, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 119355211776, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119355211776, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T144109965833Z.pickle", + "timestamp_utc": "2026-08-01T14:41:10.069356Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T144109965833Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T144109965833Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c1242d6886089a348898a9036ed5b21c8fd76d7b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000002_20260801T144109965833Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a91340f876001957682f8c14092fc0816777b996d1587ec931b0caf1d2bcb2b +size 5611799 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T144111160254Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T144111160254Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0c3dfec9f15e14bef956e4bbe306d3350cde87a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T144111160254Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5672258560, + "inactive_split_bytes_peak": 11826733568, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:11.159853Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5204593664, + "inactive_split_bytes_peak": 11252827648, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 119355211776, + "reserved_bytes_peak": 119355211776, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:10.082526Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5672258560, + "inactive_split_bytes_current_delta": 467664896, + "inactive_split_bytes_peak": 11826733568, + "inactive_split_bytes_peak_delta": 573905920, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 120179392512, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 120179392512, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 490, + "segment_count_current_delta": 1, + "segment_count_peak": 490, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T144111160254Z.pickle", + "timestamp_utc": "2026-08-01T14:41:11.261715Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T144111160254Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T144111160254Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..524e23bcf578f7254bf46f7c34d6c41d0cb43810 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000003_20260801T144111160254Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7b6699d9c47b4e8d329c91d72aea5637d116ad33959c38c4b0bc3cd7891e430 +size 5613020 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T144112594498Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T144112594498Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6a4f4dd9eec68a2855930ed78a2d480de0ce9940 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T144112594498Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 11826733568, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121003573248, + "reserved_bytes_peak": 121003573248, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T14:41:12.594083Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5672258560, + "inactive_split_bytes_peak": 11826733568, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 120179392512, + "reserved_bytes_peak": 120179392512, + "segment_count_current": 490, + "segment_count_peak": 490, + "timestamp_utc": "2026-08-01T14:41:11.274525Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_current_delta": -471859200, + "inactive_split_bytes_peak": 11826733568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121003573248, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 121003573248, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 491, + "segment_count_current_delta": 1, + "segment_count_peak": 491, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T144112594498Z.pickle", + "timestamp_utc": "2026-08-01T14:41:12.699644Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T144112594498Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T144112594498Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b254153b3020272fb2ff1128e1383527da9e8a16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000004_20260801T144112594498Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f2de61aebad2c8d83652c7d94308df889155f7c64950737616d349a5ae8a6ae +size 5611715 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T144113775043Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T144113775043Z.json new file mode 100644 index 0000000000000000000000000000000000000000..610dfa0d3249fc686a76ededb495645b3827dd14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T144113775043Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5435280384, + "inactive_split_bytes_peak": 11826733568, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121003573248, + "reserved_bytes_peak": 121003573248, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T14:41:13.774637Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 11826733568, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121003573248, + "reserved_bytes_peak": 121003573248, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T14:41:12.712763Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5435280384, + "inactive_split_bytes_current_delta": 234881024, + "inactive_split_bytes_peak": 11826733568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121003573248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121003573248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 491, + "segment_count_current_delta": 0, + "segment_count_peak": 491, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T144113775043Z.pickle", + "timestamp_utc": "2026-08-01T14:41:13.873473Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T144113775043Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T144113775043Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ef5dc98be5ecee4132d038cb7bfbe734004d68e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000005_20260801T144113775043Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56ab9f398cf8f1e359810db339d3a5e1ad4a7dfb5378a997dfee79956625ed1e +size 5612789 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T144114947065Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T144114947065Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7f5bf6a93dfb595f07bc259f0425da39273d932d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T144114947065Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 11826733568, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121003573248, + "reserved_bytes_peak": 121003573248, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T14:41:14.946657Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5435280384, + "inactive_split_bytes_peak": 11826733568, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121003573248, + "reserved_bytes_peak": 121003573248, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T14:41:13.886168Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_current_delta": -232783872, + "inactive_split_bytes_peak": 11826733568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121003573248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121003573248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 491, + "segment_count_current_delta": 0, + "segment_count_peak": 491, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T144114947065Z.pickle", + "timestamp_utc": "2026-08-01T14:41:15.051079Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T144114947065Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T144114947065Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..91c5b1776ce7a462ddeb8d6d89b41a2cd1b6aadf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000006_20260801T144114947065Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dfe10d53499704d37a218b0203649d872edefcd934b795b93e73f287cbe327f +size 5612842 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T144116116320Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T144116116320Z.json new file mode 100644 index 0000000000000000000000000000000000000000..de35f0b0d6a05eba877fce4a0bba70bb4af96573 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T144116116320Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_peak": 11826733568, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121003573248, + "reserved_bytes_peak": 121003573248, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T14:41:16.115853Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 11826733568, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121003573248, + "reserved_bytes_peak": 121003573248, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T14:41:15.066194Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_current_delta": -12582912, + "inactive_split_bytes_peak": 11826733568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121003573248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121003573248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 491, + "segment_count_current_delta": 0, + "segment_count_peak": 491, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T144116116320Z.pickle", + "timestamp_utc": "2026-08-01T14:41:16.220058Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T144116116320Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T144116116320Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..085e2ea1ebdf03da5f5995c8ad3c6d872170eb73 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000007_20260801T144116116320Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:230c672a8108b8f2a5f55cfab005aba5010aee5f54d56618efa53e5bff2f1acc +size 5612475 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T144117305619Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T144117305619Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ef594f56387bd2aeb224337044e0f6f45cfb901b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T144117305619Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 11826733568, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121003573248, + "reserved_bytes_peak": 121003573248, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T14:41:17.305222Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_peak": 11826733568, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121003573248, + "reserved_bytes_peak": 121003573248, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T14:41:16.232900Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_current_delta": 12582912, + "inactive_split_bytes_peak": 11826733568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121003573248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121003573248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 491, + "segment_count_current_delta": 0, + "segment_count_peak": 491, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T144117305619Z.pickle", + "timestamp_utc": "2026-08-01T14:41:17.403516Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T144117305619Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T144117305619Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1fedf6ab3accd59402c38bdba3986123e2a92862 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000008_20260801T144117305619Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:630ecb18b0b445f60fb48464e308c9cc43e55a6d9639d4bd5a4d9737e8058988 +size 5611500 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T144216459501Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T144216459501Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ac0dcb0bc9d639fa3c53ef135109b250d70814ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T144216459501Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 4928080896, + "inactive_split_bytes_peak": 11826733568, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121003573248, + "reserved_bytes_peak": 121003573248, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T14:42:16.459026Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 5, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 11826733568, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121003573248, + "reserved_bytes_peak": 121003573248, + "segment_count_current": 491, + "segment_count_peak": 491, + "timestamp_utc": "2026-08-01T14:41:17.415957Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 4928080896, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 11826733568, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121003573248, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121003573248, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 491, + "segment_count_current_delta": 0, + "segment_count_peak": 491, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T144216459501Z.pickle", + "timestamp_utc": "2026-08-01T14:42:16.654497Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T144216459501Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T144216459501Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aacd0c6f444d011fa7ee371dcc74303595e56489 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0005_iter000009_20260801T144216459501Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43c65482869c49551a772079b4da74906c44f3be58c825b770278cce70bf3edb +size 9655438 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T144107546772Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T144107546772Z.json new file mode 100644 index 0000000000000000000000000000000000000000..cd9e713bc2c17569ef6067b59ff7913817c2fdca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T144107546772Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 31622093824, + "active_bytes_peak": 92108908544, + "active_bytes_peak_delta": 56642572288, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92108908544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92108908544, + "device": 6, + "inactive_split_bytes_current": 5424794624, + "inactive_split_bytes_peak": 10309283840, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98272542720, + "reserved_bytes_peak": 98272542720, + "segment_count_current": 434, + "segment_count_peak": 434, + "timestamp_utc": "2026-08-01T14:41:07.546171Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 31622093824, + "allocated_bytes_peak": 92108908544, + "allocated_bytes_peak_delta": 56642572288, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:39:58.590861Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5424794624, + "inactive_split_bytes_current_delta": 5291975680, + "inactive_split_bytes_peak": 10309283840, + "inactive_split_bytes_peak_delta": 10148897792, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98272542720, + "reserved_bytes_current_delta": 59028537344, + "reserved_bytes_peak": 98272542720, + "reserved_bytes_peak_delta": 59028537344, + "segment_count_current": 434, + "segment_count_current_delta": 363, + "segment_count_peak": 434, + "segment_count_peak_delta": 363, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T144107546772Z.pickle", + "timestamp_utc": "2026-08-01T14:41:07.657407Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T144107546772Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T144107546772Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b47fcafb75b8525a4d33c94964254e56404265fb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000000_20260801T144107546772Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b592df8c35a713d384f6159d6819f730558f3c1e2e4f72b985fdd75246723708 +size 5385112 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T144108759399Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T144108759399Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ddd0a23a4f8d048fdfd17ca4a9a5bc15df90e81 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T144108759399Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 23645974528, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5443668992, + "inactive_split_bytes_peak": 11210279936, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 118885449728, + "reserved_bytes_peak": 118885449728, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T14:41:08.758992Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 23645974528, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 92108908544, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 92108908544, + "device": 6, + "inactive_split_bytes_current": 5424794624, + "inactive_split_bytes_peak": 10309283840, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98272542720, + "reserved_bytes_peak": 98272542720, + "segment_count_current": 434, + "segment_count_peak": 434, + "timestamp_utc": "2026-08-01T14:41:07.674565Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5443668992, + "inactive_split_bytes_current_delta": 18874368, + "inactive_split_bytes_peak": 11210279936, + "inactive_split_bytes_peak_delta": 900996096, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 118885449728, + "reserved_bytes_current_delta": 20612907008, + "reserved_bytes_peak": 118885449728, + "reserved_bytes_peak_delta": 20612907008, + "segment_count_current": 487, + "segment_count_current_delta": 53, + "segment_count_peak": 487, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T144108759399Z.pickle", + "timestamp_utc": "2026-08-01T14:41:08.874950Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T144108759399Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T144108759399Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3ac1c5e9667b7569d5667eb973f7bc1ad8a86656 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000001_20260801T144108759399Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:affd495be8d404b3717b44be217e07efc858455b55122c13aab880005a085459 +size 5612967 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T144109965837Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T144109965837Z.json new file mode 100644 index 0000000000000000000000000000000000000000..780b47a3ad13e2c4d60b986b73f4a66e7d656b2e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T144109965837Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5674355712, + "inactive_split_bytes_peak": 11210279936, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:09.965432Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5443668992, + "inactive_split_bytes_peak": 11210279936, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 118885449728, + "reserved_bytes_peak": 118885449728, + "segment_count_current": 487, + "segment_count_peak": 487, + "timestamp_utc": "2026-08-01T14:41:08.894543Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5674355712, + "inactive_split_bytes_current_delta": 230686720, + "inactive_split_bytes_peak": 11210279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119709630464, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 119709630464, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 488, + "segment_count_current_delta": 1, + "segment_count_peak": 488, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T144109965837Z.pickle", + "timestamp_utc": "2026-08-01T14:41:10.070760Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T144109965837Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T144109965837Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4475700c0801769a478006475aca13fd710d9d8f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000002_20260801T144109965837Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80e568aba2b6b91ced8320158022bd790bcc9179b0eefbe7440bb8eed35911c0 +size 5611553 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T144111160292Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T144111160292Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dd1a096f1db9826f8aebcdab99ae0ea06a453ff1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T144111160292Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5437377536, + "inactive_split_bytes_peak": 11210279936, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:11.159871Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5674355712, + "inactive_split_bytes_peak": 11210279936, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:10.084115Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5437377536, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 11210279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119709630464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119709630464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T144111160292Z.pickle", + "timestamp_utc": "2026-08-01T14:41:11.261369Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T144111160292Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T144111160292Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6a75c9acdafabdcb36db4130c3ba28e0e29651e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000003_20260801T144111160292Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f55077c4be2c91de0caec87e43961cb193ebb55ebb2ca1dd908a1725aeb27bf +size 5612578 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T144112594518Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T144112594518Z.json new file mode 100644 index 0000000000000000000000000000000000000000..98bc5ad1000c1e6bab2feccd757bac228f8ab309 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T144112594518Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 11210279936, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:12.594106Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5437377536, + "inactive_split_bytes_peak": 11210279936, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:11.274255Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_current_delta": -236978176, + "inactive_split_bytes_peak": 11210279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119709630464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119709630464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T144112594518Z.pickle", + "timestamp_utc": "2026-08-01T14:41:12.697257Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T144112594518Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T144112594518Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..44ec09621fd765881df870fad55206587ef27e35 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000004_20260801T144112594518Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0ccf7691ee17fee4a0ed2f39c765c7bb1d9c2af71110c5f22f7ca4f37544d98 +size 5611158 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T144113775061Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T144113775061Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4be1865c780b67c75cd035a489b80aa2d87a3a95 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T144113775061Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 11210279936, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:13.774662Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 11210279936, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:12.710115Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11210279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119709630464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119709630464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T144113775061Z.pickle", + "timestamp_utc": "2026-08-01T14:41:13.873717Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T144113775061Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T144113775061Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b85fe15b930b8414bb3bc8f286488b23f2532f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000005_20260801T144113775061Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c60e54fa4b18ef0099e9454cbefa6926e762444cfb09bf867a7c405f1e16b19e +size 5612152 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T144114947121Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T144114947121Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3d05af3b11c3fca07fa6eb1268b2561ee271e043 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T144114947121Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 11210279936, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:14.946697Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5200399360, + "inactive_split_bytes_peak": 11210279936, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:13.886473Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11210279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119709630464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119709630464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T144114947121Z.pickle", + "timestamp_utc": "2026-08-01T14:41:15.046673Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T144114947121Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T144114947121Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0718744cfbdc3830501c628080f743dd3cb2f9eb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000006_20260801T144114947121Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8fb664d81e10b668b35cb2fc115896ae6c2184ead4869574ad77af2590c9bfa +size 5612303 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T144116116280Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T144116116280Z.json new file mode 100644 index 0000000000000000000000000000000000000000..563487eb5c675dd2e75f49db6e8c54b9965ea2b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T144116116280Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_peak": 11210279936, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:16.115853Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 11210279936, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:15.059211Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_current_delta": -12582912, + "inactive_split_bytes_peak": 11210279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119709630464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119709630464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T144116116280Z.pickle", + "timestamp_utc": "2026-08-01T14:41:16.221288Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T144116116280Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T144116116280Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..609ef355b595d515bb6f0329b03036229e6c8abd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000007_20260801T144116116280Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b621cae2344e387e7d189c7f029e8e41a6268715e46d8fa442e4793ffca2757f +size 5612021 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T144117305657Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T144117305657Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5efd3b113bde5fd3b7c3cf7cbf86f8271571b57f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T144117305657Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549056, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 11210279936, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:17.305244Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549056, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5189913600, + "inactive_split_bytes_peak": 11210279936, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:16.236645Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_current_delta": 12582912, + "inactive_split_bytes_peak": 11210279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119709630464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119709630464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T144117305657Z.pickle", + "timestamp_utc": "2026-08-01T14:41:17.403592Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T144117305657Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T144117305657Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4dbfc585367e4722f66ee999e7d945e2703aeaf8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000008_20260801T144117305657Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff4b1209450a57a5963e7b2a577de49bc1dd67f06a56ae53828ad2eb3d9f4d54 +size 5611116 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T144216459488Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T144216459488Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a5f5ed5e72d0ff683eeab0c64a814a6db696ba94 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T144216459488Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906112, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115754883072, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906112, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66802906112, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 4928080896, + "inactive_split_bytes_peak": 11210279936, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:42:16.459017Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906112, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115754883072, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549056, + "active_bytes_peak": 115754883072, + "allocated_bytes_current": 66853549056, + "allocated_bytes_peak": 115754883072, + "device": 6, + "inactive_split_bytes_current": 5202496512, + "inactive_split_bytes_peak": 11210279936, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 119709630464, + "reserved_bytes_peak": 119709630464, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:17.415913Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 4928080896, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 11210279936, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 119709630464, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119709630464, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T144216459488Z.pickle", + "timestamp_utc": "2026-08-01T14:42:16.652549Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T144216459488Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T144216459488Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cc6da81832cbf613e76c1e5b6b5b13e4fec5dbb4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0006_iter000009_20260801T144216459488Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e3712c8a646f173adb2b318cf6c4be633ad652b9aa095cc295a2694f790028d +size 9640609 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T144107550653Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T144107550653Z.json new file mode 100644 index 0000000000000000000000000000000000000000..061e914464ea760ceeecd815a6c69339aedea081 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T144107550653Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 31622094336, + "active_bytes_peak": 92108908544, + "active_bytes_peak_delta": 56642572288, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 92108908544, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 92108908544, + "device": 7, + "inactive_split_bytes_current": 5659675136, + "inactive_split_bytes_peak": 10389252608, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 98507423744, + "reserved_bytes_peak": 98507423744, + "segment_count_current": 435, + "segment_count_peak": 435, + "timestamp_utc": "2026-08-01T14:41:07.550168Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 31622094336, + "allocated_bytes_peak": 92108908544, + "allocated_bytes_peak_delta": 56642572288, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:39:58.592789Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5659675136, + "inactive_split_bytes_current_delta": 5526856192, + "inactive_split_bytes_peak": 10389252608, + "inactive_split_bytes_peak_delta": 10228866560, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 98507423744, + "reserved_bytes_current_delta": 59263418368, + "reserved_bytes_peak": 98507423744, + "reserved_bytes_peak_delta": 59263418368, + "segment_count_current": 435, + "segment_count_current_delta": 364, + "segment_count_peak": 435, + "segment_count_peak_delta": 364, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T144107550653Z.pickle", + "timestamp_utc": "2026-08-01T14:41:07.661805Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T144107550653Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T144107550653Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..915266f9feabe12303a0f39d2ae2418951ecfda0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000000_20260801T144107550653Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcfd51d0f81a94d75a9b6441ea8af4ee4fbb0603bd7f7b92b91dac54987b71c2 +size 5386951 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T144108759412Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T144108759412Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ceda20fba1ad5f31d7d067a7e668c149783e4619 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T144108759412Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115753572864, + "active_bytes_peak_delta": 23644664320, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5208787456, + "inactive_split_bytes_peak": 10564239360, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119120330752, + "reserved_bytes_peak": 119120330752, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:08.759011Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115753572864, + "allocated_bytes_peak_delta": 23644664320, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 92108908544, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 92108908544, + "device": 7, + "inactive_split_bytes_current": 5659675136, + "inactive_split_bytes_peak": 10389252608, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 98507423744, + "reserved_bytes_peak": 98507423744, + "segment_count_current": 435, + "segment_count_peak": 435, + "timestamp_utc": "2026-08-01T14:41:07.678958Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5208787456, + "inactive_split_bytes_current_delta": -450887680, + "inactive_split_bytes_peak": 10564239360, + "inactive_split_bytes_peak_delta": 174986752, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119120330752, + "reserved_bytes_current_delta": 20612907008, + "reserved_bytes_peak": 119120330752, + "reserved_bytes_peak_delta": 20612907008, + "segment_count_current": 488, + "segment_count_current_delta": 53, + "segment_count_peak": 488, + "segment_count_peak_delta": 53, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T144108759412Z.pickle", + "timestamp_utc": "2026-08-01T14:41:08.869142Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T144108759412Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T144108759412Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..406457f16849c42315ff5a8785d2c68bb452b978 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000001_20260801T144108759412Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d25de7f0cac6d590040f7119bb0ba3ff24f03533e1b52fa5cfe43b5bca174b3 +size 5614986 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T144109965834Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T144109965834Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e0b4002e3e0b67e5a39a126019a6e67e72c68647 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T144109965834Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115753572864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5206690304, + "inactive_split_bytes_peak": 10564239360, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119120330752, + "reserved_bytes_peak": 119120330752, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:09.965437Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115753572864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5208787456, + "inactive_split_bytes_peak": 10564239360, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119120330752, + "reserved_bytes_peak": 119120330752, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:08.884502Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5206690304, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 10564239360, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119120330752, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119120330752, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 488, + "segment_count_current_delta": 0, + "segment_count_peak": 488, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T144109965834Z.pickle", + "timestamp_utc": "2026-08-01T14:41:10.070537Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T144109965834Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T144109965834Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..b49740872d9015b4f5315f739ccb1954c57fd8c9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000002_20260801T144109965834Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3f15305dc02cc188ce64e8126d5bed223299eddb498c07830b112bd6988e1b4 +size 5613256 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T144111160392Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T144111160392Z.json new file mode 100644 index 0000000000000000000000000000000000000000..af046aaf12fcb5eca417b0c2697c3a0979e4c5e1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T144111160392Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115753572864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5674355200, + "inactive_split_bytes_peak": 11052359680, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:11.159959Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115753572864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5206690304, + "inactive_split_bytes_peak": 10564239360, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119120330752, + "reserved_bytes_peak": 119120330752, + "segment_count_current": 488, + "segment_count_peak": 488, + "timestamp_utc": "2026-08-01T14:41:10.083741Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5674355200, + "inactive_split_bytes_current_delta": 467664896, + "inactive_split_bytes_peak": 11052359680, + "inactive_split_bytes_peak_delta": 488120320, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119944511488, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 119944511488, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 489, + "segment_count_current_delta": 1, + "segment_count_peak": 489, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T144111160392Z.pickle", + "timestamp_utc": "2026-08-01T14:41:11.261937Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T144111160392Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T144111160392Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1c3a55e133d59ec2cc7dff949a7bc4de763045a7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000003_20260801T144111160392Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44fd66756970c43031c561340f2bb55a78956434fe47bbeb6663e8aa2920768c +size 5614448 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T144112594548Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T144112594548Z.json new file mode 100644 index 0000000000000000000000000000000000000000..e564235917b46c60aad8039a4a2a495252bde34e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T144112594548Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115753572864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5202496000, + "inactive_split_bytes_peak": 11052359680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:12.594110Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115753572864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5674355200, + "inactive_split_bytes_peak": 11052359680, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:11.274665Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5202496000, + "inactive_split_bytes_current_delta": -471859200, + "inactive_split_bytes_peak": 11052359680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119944511488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119944511488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T144112594548Z.pickle", + "timestamp_utc": "2026-08-01T14:41:12.697626Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T144112594548Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T144112594548Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0a9e804bd6224c4b39d5e83fbe1378a273217d21 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000004_20260801T144112594548Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f867af424688f3b532575437b095d35fba425291e30d404d80fbd2f5003945f8 +size 5612863 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T144113775113Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T144113775113Z.json new file mode 100644 index 0000000000000000000000000000000000000000..beeaf6f6a6886dfa5bab1e179b00c1f1d8ca21de --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T144113775113Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115753572864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5202496000, + "inactive_split_bytes_peak": 11052359680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:13.774723Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115753572864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5202496000, + "inactive_split_bytes_peak": 11052359680, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:12.710535Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5202496000, + "inactive_split_bytes_current_delta": 0, + "inactive_split_bytes_peak": 11052359680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119944511488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119944511488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T144113775113Z.pickle", + "timestamp_utc": "2026-08-01T14:41:13.874799Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T144113775113Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T144113775113Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dfb2cae6dcad956ba8a56c4fe8ece3a3f5724fb4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000005_20260801T144113775113Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef38e9b999851bacd2fa07cfe48a38c5c32d089532f6497f9a64dd031976847e +size 5613925 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T144114947175Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T144114947175Z.json new file mode 100644 index 0000000000000000000000000000000000000000..27ff1c1385aee54f04c23945ff7574db75e25d31 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T144114947175Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115753572864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5204593152, + "inactive_split_bytes_peak": 11052359680, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:14.946767Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115753572864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5202496000, + "inactive_split_bytes_peak": 11052359680, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:13.887480Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5204593152, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 11052359680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119944511488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119944511488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T144114947175Z.pickle", + "timestamp_utc": "2026-08-01T14:41:15.047970Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T144114947175Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T144114947175Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..317bdd570495e772dd0620d78e7ef3e1fe70b801 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000006_20260801T144114947175Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cac9df847ac202cd341b9795516df82e9b0794d524484862c235408ad67b14a +size 5614044 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T144116116345Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T144116116345Z.json new file mode 100644 index 0000000000000000000000000000000000000000..be7b4713ccd33a995fd7d13ef60d27a0fd23470a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T144116116345Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115753572864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5192010240, + "inactive_split_bytes_peak": 11052359680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:16.115915Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115753572864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5204593152, + "inactive_split_bytes_peak": 11052359680, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:15.060612Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5192010240, + "inactive_split_bytes_current_delta": -12582912, + "inactive_split_bytes_peak": 11052359680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119944511488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119944511488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T144116116345Z.pickle", + "timestamp_utc": "2026-08-01T14:41:16.223826Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T144116116345Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T144116116345Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..554bcea366051c55a7d90dfdede25127cef530f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000007_20260801T144116116345Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16fbf4a607c65ed005c239c5660ba3f01c87f6afb83e93bb5df96f92b542068c +size 5613793 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T144117305701Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T144117305701Z.json new file mode 100644 index 0000000000000000000000000000000000000000..90346967b43d8e39e310b9ebec5c5f1296d47cdf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T144117305701Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66853549568, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115753572864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5204593152, + "inactive_split_bytes_peak": 11052359680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:17.305304Z", + "world_size": 8 + }, + "allocated_bytes_current": 66853549568, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115753572864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5192010240, + "inactive_split_bytes_peak": 11052359680, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:16.238763Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 5204593152, + "inactive_split_bytes_current_delta": 12582912, + "inactive_split_bytes_peak": 11052359680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119944511488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119944511488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T144117305701Z.pickle", + "timestamp_utc": "2026-08-01T14:41:17.409111Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T144117305701Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T144117305701Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..813c75ea91e388bf2426b3bfd1890d42df540aa1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000008_20260801T144117305701Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf9447b35cc0caa8e139e74d65a64511d0eded2e4f3bef6557b267e8de352cae +size 5612827 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T144216459487Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T144216459487Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3fba57c181cf0ad9666d247943a6f27d531f2abd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T144216459487Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 66802906624, + "active_bytes_current_delta": -50642944, + "active_bytes_peak": 115753572864, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 66802906624, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66802906624, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 4930177536, + "inactive_split_bytes_peak": 11052359680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:42:16.459022Z", + "world_size": 8 + }, + "allocated_bytes_current": 66802906624, + "allocated_bytes_current_delta": -50642944, + "allocated_bytes_peak": 115753572864, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 66853549568, + "active_bytes_peak": 115753572864, + "allocated_bytes_current": 66853549568, + "allocated_bytes_peak": 115753572864, + "device": 7, + "inactive_split_bytes_current": 5204593152, + "inactive_split_bytes_peak": 11052359680, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 119944511488, + "reserved_bytes_peak": 119944511488, + "segment_count_current": 489, + "segment_count_peak": 489, + "timestamp_utc": "2026-08-01T14:41:17.424389Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 4930177536, + "inactive_split_bytes_current_delta": -274415616, + "inactive_split_bytes_peak": 11052359680, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 119944511488, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 119944511488, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 489, + "segment_count_current_delta": 0, + "segment_count_peak": 489, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T144216459487Z.pickle", + "timestamp_utc": "2026-08-01T14:42:16.665907Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T144216459487Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T144216459487Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..080c265ecce278daae64b00bf54f88280b2473f7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off/snapshot_iteration_rank0007_iter000009_20260801T144216459487Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7016390e9355eff1fb732ba09f4d5ca77a3bed424ff7b1ee73e4963bf6ec70a0 +size 10211492 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/_SUCCESS b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/_SUCCESS new file mode 100644 index 0000000000000000000000000000000000000000..ff43ca409150a5479415981803955ee1d40e4d16 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/_SUCCESS @@ -0,0 +1 @@ +SUCCESS diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/run_metadata.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/run_metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..2507a86c732099fa2511c23623af786b8b030ebe --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/run_metadata.txt @@ -0,0 +1,14 @@ +cuda_graph: off +seq_length: 4096 +capacity: 1.50 +expert_dispatch_overlap: on +expert_dispatch_recompute: on +global_batch_size: 32 +train_iters: 10 +master_port: 6047 +expected_pickles: 80 +actual_pickles: 80 +status: SUCCESS +duration_s: 236 +log_file: /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on.log +result_dir: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T144503143534Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T144503143534Z.json new file mode 100644 index 0000000000000000000000000000000000000000..05e8def533e55f9cc01b9566d0efebb857eb5a9f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T144503143534Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 37209103360, + "active_bytes_peak": 93048106496, + "active_bytes_peak_delta": 57581770240, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 93048106496, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 91991261184, + "device": 0, + "inactive_split_bytes_current": 6072617984, + "inactive_split_bytes_peak": 12544613376, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 97439973376, + "reserved_bytes_peak": 97439973376, + "segment_count_current": 380, + "segment_count_peak": 380, + "timestamp_utc": "2026-08-01T14:45:03.142963Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 37209103360, + "allocated_bytes_peak": 91991261184, + "allocated_bytes_peak_delta": 56524924928, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 0, + "inactive_split_bytes_current": 134916096, + "inactive_split_bytes_peak": 160631808, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 39246102528, + "reserved_bytes_peak": 39246102528, + "segment_count_current": 72, + "segment_count_peak": 72, + "timestamp_utc": "2026-08-01T14:43:54.202260Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6072617984, + "inactive_split_bytes_current_delta": 5937701888, + "inactive_split_bytes_peak": 12544613376, + "inactive_split_bytes_peak_delta": 12383981568, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 97439973376, + "reserved_bytes_current_delta": 58193870848, + "reserved_bytes_peak": 97439973376, + "reserved_bytes_peak_delta": 58193870848, + "segment_count_current": 380, + "segment_count_current_delta": 308, + "segment_count_peak": 380, + "segment_count_peak_delta": 308, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000000_20260801T144503143534Z.pickle", + "timestamp_utc": "2026-08-01T14:45:03.262890Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T144503143534Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T144503143534Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6ac937ad946f6cfca391db42c94c0031f87c48f4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000000_20260801T144503143534Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a5f7777191fce9f73bf22ddbe89737d1e1a9f6348ad04f28c9e729d8360e219 +size 5953677 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T144504559759Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T144504559759Z.json new file mode 100644 index 0000000000000000000000000000000000000000..92e7981ecd1ade2faecec5cdce0f94a66689996a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T144504559759Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116000169984, + "active_bytes_peak_delta": 22952063488, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116000169984, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115647848448, + "device": 0, + "inactive_split_bytes_current": 8243170304, + "inactive_split_bytes_peak": 13834361856, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:04.559344Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115647848448, + "allocated_bytes_peak_delta": 23656587264, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 93048106496, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 91991261184, + "device": 0, + "inactive_split_bytes_current": 6072617984, + "inactive_split_bytes_peak": 12544613376, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 97439973376, + "reserved_bytes_peak": 97439973376, + "segment_count_current": 380, + "segment_count_peak": 380, + "timestamp_utc": "2026-08-01T14:45:03.283091Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8243170304, + "inactive_split_bytes_current_delta": 2170552320, + "inactive_split_bytes_peak": 13834361856, + "inactive_split_bytes_peak_delta": 1289748480, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_current_delta": 24784142336, + "reserved_bytes_peak": 122224115712, + "reserved_bytes_peak_delta": 24784142336, + "segment_count_current": 445, + "segment_count_current_delta": 65, + "segment_count_peak": 445, + "segment_count_peak_delta": 65, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000001_20260801T144504559759Z.pickle", + "timestamp_utc": "2026-08-01T14:45:04.675875Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T144504559759Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T144504559759Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..93fe5ca20caff165d439f06969eeead4a010fb50 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000001_20260801T144504559759Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c75b123e4204d1e7f4989ec145187ce828bf9993cb76692768aa4c46d61f147a +size 6126333 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T144505924362Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T144505924362Z.json new file mode 100644 index 0000000000000000000000000000000000000000..843c572aa3bae965b26f94edef361c793446b313 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T144505924362Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707070976, + "active_bytes_peak_delta": 706900992, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115650106368, + "device": 0, + "inactive_split_bytes_current": 9243511808, + "inactive_split_bytes_peak": 15475497984, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:05.923959Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115650106368, + "allocated_bytes_peak_delta": 2257920, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116000169984, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115647848448, + "device": 0, + "inactive_split_bytes_current": 8243170304, + "inactive_split_bytes_peak": 13834361856, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:04.690151Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9243511808, + "inactive_split_bytes_current_delta": 1000341504, + "inactive_split_bytes_peak": 15475497984, + "inactive_split_bytes_peak_delta": 1641136128, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122224115712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 445, + "segment_count_current_delta": 0, + "segment_count_peak": 445, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000002_20260801T144505924362Z.pickle", + "timestamp_utc": "2026-08-01T14:45:06.031722Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T144505924362Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T144505924362Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f7d5da23f10319bdea16288fb1c6ff41071e2ad9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000002_20260801T144505924362Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:895c10c5bc2556aa9e8f210a296ef16fe36999f34a56985345c88630a52231ed +size 6626251 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T144507273644Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T144507273644Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5443884d74fb0f6bbfc53ac250c6d6c853b1a7c5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T144507273644Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707070976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115650106368, + "device": 0, + "inactive_split_bytes_current": 8478051328, + "inactive_split_bytes_peak": 15475497984, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:07.273240Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115650106368, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115650106368, + "device": 0, + "inactive_split_bytes_current": 9243511808, + "inactive_split_bytes_peak": 15475497984, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:06.045403Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8478051328, + "inactive_split_bytes_current_delta": -765460480, + "inactive_split_bytes_peak": 15475497984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122224115712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 445, + "segment_count_current_delta": 0, + "segment_count_peak": 445, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000003_20260801T144507273644Z.pickle", + "timestamp_utc": "2026-08-01T14:45:07.383624Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T144507273644Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T144507273644Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..63c478d2c8a8df500478f081109911e456f0a46f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000003_20260801T144507273644Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eaacd7b5a30369910f4179a2caa51980a4ec8185961f899d90f88320e00bc9be +size 6625473 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T144508911446Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T144508911446Z.json new file mode 100644 index 0000000000000000000000000000000000000000..75ad5a42d2c488b3a89d8fec435ba9177f68d6e5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T144508911446Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707070976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115650883584, + "device": 0, + "inactive_split_bytes_current": 8654212096, + "inactive_split_bytes_peak": 15475497984, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:08.911025Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115650883584, + "allocated_bytes_peak_delta": 777216, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115650106368, + "device": 0, + "inactive_split_bytes_current": 8478051328, + "inactive_split_bytes_peak": 15475497984, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:07.397568Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8654212096, + "inactive_split_bytes_current_delta": 176160768, + "inactive_split_bytes_peak": 15475497984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122224115712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 445, + "segment_count_current_delta": 0, + "segment_count_peak": 445, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000004_20260801T144508911446Z.pickle", + "timestamp_utc": "2026-08-01T14:45:09.020840Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T144508911446Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T144508911446Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cf72131468304698e2c49b7e0a53c71eb15b69c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000004_20260801T144508911446Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a63f8ca0d40e75072a03d5aa81c7835c4e6bd33bda147bd6d5226d6bd5b69606 +size 6643416 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T144510252590Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T144510252590Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0511d04333e3d1aef9bd45652bd968dc411d59fc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T144510252590Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707070976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115653394944, + "device": 0, + "inactive_split_bytes_current": 9476295680, + "inactive_split_bytes_peak": 15475497984, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:10.252169Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115653394944, + "allocated_bytes_peak_delta": 2511360, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115650883584, + "device": 0, + "inactive_split_bytes_current": 8654212096, + "inactive_split_bytes_peak": 15475497984, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:09.034815Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9476295680, + "inactive_split_bytes_current_delta": 822083584, + "inactive_split_bytes_peak": 15475497984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122224115712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 445, + "segment_count_current_delta": 0, + "segment_count_peak": 445, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000005_20260801T144510252590Z.pickle", + "timestamp_utc": "2026-08-01T14:45:10.361960Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T144510252590Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T144510252590Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1071dca3ea40e269a319b16e47c8280b84777b29 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000005_20260801T144510252590Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d825af6213de61698d7128140efcf4a87038dd34ab2185509611db99c23ef4ca +size 6642224 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T144511613590Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T144511613590Z.json new file mode 100644 index 0000000000000000000000000000000000000000..17cc62a1f1c0dbfedb1df40ca93b8fb98697104e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T144511613590Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707070976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115653394944, + "device": 0, + "inactive_split_bytes_current": 8419331072, + "inactive_split_bytes_peak": 15475497984, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:11.613182Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115653394944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115653394944, + "device": 0, + "inactive_split_bytes_current": 9476295680, + "inactive_split_bytes_peak": 15475497984, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:10.375860Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 8419331072, + "inactive_split_bytes_current_delta": -1056964608, + "inactive_split_bytes_peak": 15475497984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122224115712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 445, + "segment_count_current_delta": 0, + "segment_count_peak": 445, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000006_20260801T144511613590Z.pickle", + "timestamp_utc": "2026-08-01T14:45:11.718958Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T144511613590Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T144511613590Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5856c354bcf2f6f52f808d85372130faa83ca84b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000006_20260801T144511613590Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2784717bfd1750954243e3ecadb6a7c98221e72bc172037c04c112983855b88e +size 6642570 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T144513084111Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T144513084111Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b0a71c98ef9bd2563bfe5db0369cd6c2004ca057 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T144513084111Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707070976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115656286208, + "device": 0, + "inactive_split_bytes_current": 6777261056, + "inactive_split_bytes_peak": 15475497984, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:13.083699Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115656286208, + "allocated_bytes_peak_delta": 2891264, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115653394944, + "device": 0, + "inactive_split_bytes_current": 8419331072, + "inactive_split_bytes_peak": 15475497984, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:11.732465Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 6777261056, + "inactive_split_bytes_current_delta": -1642070016, + "inactive_split_bytes_peak": 15475497984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122224115712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 445, + "segment_count_current_delta": 0, + "segment_count_peak": 445, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000007_20260801T144513084111Z.pickle", + "timestamp_utc": "2026-08-01T14:45:13.191825Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T144513084111Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T144513084111Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..065b515a006149fcb16d3cc7664bd86bcc958ab9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000007_20260801T144513084111Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f483f3784249cfb9664004d4409acfcfdb4fcb94277fd014fa97c26e774eb353 +size 6642376 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T144514411834Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T144514411834Z.json new file mode 100644 index 0000000000000000000000000000000000000000..10b90fd58f003fe1a8d6e3dc95402bc547818dd7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T144514411834Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440558592, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707070976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115656286208, + "device": 0, + "inactive_split_bytes_current": 7892945920, + "inactive_split_bytes_peak": 15475497984, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:14.411433Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440558592, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115656286208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115656286208, + "device": 0, + "inactive_split_bytes_current": 6777261056, + "inactive_split_bytes_peak": 15475497984, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:13.205591Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 7892945920, + "inactive_split_bytes_current_delta": 1115684864, + "inactive_split_bytes_peak": 15475497984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122224115712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 445, + "segment_count_current_delta": 0, + "segment_count_peak": 445, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000008_20260801T144514411834Z.pickle", + "timestamp_utc": "2026-08-01T14:45:14.517217Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T144514411834Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T144514411834Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f443384eeb6c0550f8e3299b86c9b351ae415b4d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000008_20260801T144514411834Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99d94095c3faa09b2751ed5e4ad26dc02f8714a6560f8d0b36fd09fa6cd65edc +size 6641857 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T144614834126Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T144614834126Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d192f03199ee6b68084908be26d11a928a648f33 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T144614834126Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166269440, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 116707070976, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166269440, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 61166269440, + "allocated_bytes_peak": 115656286208, + "device": 0, + "inactive_split_bytes_current": 9119779840, + "inactive_split_bytes_peak": 15475497984, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:46:14.833660Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166269440, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 115656286208, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440558592, + "active_bytes_peak": 116707070976, + "allocated_bytes_current": 72440558592, + "allocated_bytes_peak": 115656286208, + "device": 0, + "inactive_split_bytes_current": 7892945920, + "inactive_split_bytes_peak": 15475497984, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_peak": 122224115712, + "segment_count_current": 445, + "segment_count_peak": 445, + "timestamp_utc": "2026-08-01T14:45:14.530736Z", + "world_size": 8 + }, + "device": 0, + "device_delta": 0, + "inactive_split_bytes_current": 9119779840, + "inactive_split_bytes_current_delta": 1226833920, + "inactive_split_bytes_peak": 15475497984, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 0, + "rank_delta": 0, + "reserved_bytes_current": 122224115712, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122224115712, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 445, + "segment_count_current_delta": 0, + "segment_count_peak": 445, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0000_iter000009_20260801T144614834126Z.pickle", + "timestamp_utc": "2026-08-01T14:46:15.038776Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T144614834126Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T144614834126Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4aebb23da3c0cda249b62dd9d04a5f036f75fbe8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0000_iter000009_20260801T144614834126Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:178145f4f5e31f76ae7e8bc7544c730b524a6340dfd43106ed8a1775a9266301 +size 10543611 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T144503143982Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T144503143982Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5d6ec091a42833dbffefb2d97f0f9a9c7799717a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T144503143982Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 92689831424, + "active_bytes_peak_delta": 57223495168, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92689831424, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 92002228224, + "device": 1, + "inactive_split_bytes_current": 5989239808, + "inactive_split_bytes_peak": 12146344448, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 97966358528, + "reserved_bytes_peak": 97966358528, + "segment_count_current": 383, + "segment_count_peak": 383, + "timestamp_utc": "2026-08-01T14:45:03.143300Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 92002228224, + "allocated_bytes_peak_delta": 56535891968, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 1, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:43:54.202060Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 5989239808, + "inactive_split_bytes_current_delta": 5856420864, + "inactive_split_bytes_peak": 12146344448, + "inactive_split_bytes_peak_delta": 11985958400, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 97966358528, + "reserved_bytes_current_delta": 58722353152, + "reserved_bytes_peak": 97966358528, + "reserved_bytes_peak_delta": 58722353152, + "segment_count_current": 383, + "segment_count_current_delta": 312, + "segment_count_peak": 383, + "segment_count_peak_delta": 312, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000000_20260801T144503143982Z.pickle", + "timestamp_utc": "2026-08-01T14:45:03.288683Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T144503143982Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T144503143982Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ec45325af04245d6f5e1f7ad4016ab819ae1f49f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000000_20260801T144503143982Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68cda7bc089d0501e16ffdffcf365b005a3f74b862e5d52b1cdd1e9cec158184 +size 5945403 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T144504559824Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T144504559824Z.json new file mode 100644 index 0000000000000000000000000000000000000000..12f092e1c9373bc5b3d27c9db1756c87103ff5ef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T144504559824Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 115993244672, + "active_bytes_peak_delta": 23303413248, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115993244672, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115649269760, + "device": 1, + "inactive_split_bytes_current": 8281426944, + "inactive_split_bytes_peak": 14590888960, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 121397837824, + "reserved_bytes_peak": 121397837824, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:04.559416Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115649269760, + "allocated_bytes_peak_delta": 23647041536, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92689831424, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 92002228224, + "device": 1, + "inactive_split_bytes_current": 5989239808, + "inactive_split_bytes_peak": 12146344448, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 97966358528, + "reserved_bytes_peak": 97966358528, + "segment_count_current": 383, + "segment_count_peak": 383, + "timestamp_utc": "2026-08-01T14:45:03.319775Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8281426944, + "inactive_split_bytes_current_delta": 2292187136, + "inactive_split_bytes_peak": 14590888960, + "inactive_split_bytes_peak_delta": 2444544512, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 121397837824, + "reserved_bytes_current_delta": 23431479296, + "reserved_bytes_peak": 121397837824, + "reserved_bytes_peak_delta": 23431479296, + "segment_count_current": 440, + "segment_count_current_delta": 57, + "segment_count_peak": 440, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000001_20260801T144504559824Z.pickle", + "timestamp_utc": "2026-08-01T14:45:04.676554Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T144504559824Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T144504559824Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7e4459318ce64076efa329c295590ada438c77c8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000001_20260801T144504559824Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21b9c1b4c6de493fa5676cea6a6d98a925c4733d09cf7b9153ed80a702fac2fd +size 6090461 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T144505924452Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T144505924452Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8cc1f8b98ba8f230041c62f0d9639a7cc9898eed --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T144505924452Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116357483008, + "active_bytes_peak_delta": 364238336, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116357483008, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115652839936, + "device": 1, + "inactive_split_bytes_current": 8042351616, + "inactive_split_bytes_peak": 15551758336, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 121397837824, + "reserved_bytes_peak": 121397837824, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:05.924040Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115652839936, + "allocated_bytes_peak_delta": 3570176, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 115993244672, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115649269760, + "device": 1, + "inactive_split_bytes_current": 8281426944, + "inactive_split_bytes_peak": 14590888960, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 121397837824, + "reserved_bytes_peak": 121397837824, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:04.694239Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8042351616, + "inactive_split_bytes_current_delta": -239075328, + "inactive_split_bytes_peak": 15551758336, + "inactive_split_bytes_peak_delta": 960869376, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 121397837824, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121397837824, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 440, + "segment_count_current_delta": 0, + "segment_count_peak": 440, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000002_20260801T144505924452Z.pickle", + "timestamp_utc": "2026-08-01T14:45:06.033839Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T144505924452Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T144505924452Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0e4b8b225286e9497326fd081c3dee8c38c62208 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000002_20260801T144505924452Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e79a9ed1a6347622cf194e98bd51a1549f2fd8984e86cf22126288fdae0c6500 +size 6026979 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T144507273715Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T144507273715Z.json new file mode 100644 index 0000000000000000000000000000000000000000..338b8fd10584e326f1bad80ae7f20253e4136713 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T144507273715Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116357483008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116357483008, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115652839936, + "device": 1, + "inactive_split_bytes_current": 9099316224, + "inactive_split_bytes_peak": 15766232064, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:07.273309Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115652839936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116357483008, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115652839936, + "device": 1, + "inactive_split_bytes_current": 8042351616, + "inactive_split_bytes_peak": 15551758336, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 121397837824, + "reserved_bytes_peak": 121397837824, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:06.048960Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 9099316224, + "inactive_split_bytes_current_delta": 1056964608, + "inactive_split_bytes_peak": 15766232064, + "inactive_split_bytes_peak_delta": 214473728, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122222018560, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 122222018560, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 441, + "segment_count_current_delta": 1, + "segment_count_peak": 441, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000003_20260801T144507273715Z.pickle", + "timestamp_utc": "2026-08-01T14:45:07.384844Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T144507273715Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T144507273715Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7336bdb439654e27401c10263198656b9e0cecd5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000003_20260801T144507273715Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecb02b03a6c324012d3dd4c8240198c4b647eb005e9b5e3e8bd27cdc994d2859 +size 6101295 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T144508911490Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T144508911490Z.json new file mode 100644 index 0000000000000000000000000000000000000000..848849e49fe8ec425c957660bb0d3ae18649492c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T144508911490Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116357483008, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116357483008, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115652839936, + "device": 1, + "inactive_split_bytes_current": 8749091840, + "inactive_split_bytes_peak": 15766232064, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:08.911079Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115652839936, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116357483008, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115652839936, + "device": 1, + "inactive_split_bytes_current": 9099316224, + "inactive_split_bytes_peak": 15766232064, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:07.399666Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8749091840, + "inactive_split_bytes_current_delta": -350224384, + "inactive_split_bytes_peak": 15766232064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122222018560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122222018560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000004_20260801T144508911490Z.pickle", + "timestamp_utc": "2026-08-01T14:45:09.022515Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T144508911490Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T144508911490Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ad8eb525d78b9f12877d8ddf015da52064428d02 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000004_20260801T144508911490Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:352f40621c67f2860781e9f991719489ab56e7075bfc64b820a9a7dfbf5a457e +size 6640610 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T144510252601Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T144510252601Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67c0a8c25b4deafe9674caa6c99da402e068db24 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T144510252601Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116358348800, + "active_bytes_peak_delta": 865792, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116358348800, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115653705728, + "device": 1, + "inactive_split_bytes_current": 7690030080, + "inactive_split_bytes_peak": 15766232064, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:10.252192Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115653705728, + "allocated_bytes_peak_delta": 865792, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116357483008, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115652839936, + "device": 1, + "inactive_split_bytes_current": 8749091840, + "inactive_split_bytes_peak": 15766232064, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:09.037658Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 7690030080, + "inactive_split_bytes_current_delta": -1059061760, + "inactive_split_bytes_peak": 15766232064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122222018560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122222018560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000005_20260801T144510252601Z.pickle", + "timestamp_utc": "2026-08-01T14:45:10.361853Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T144510252601Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T144510252601Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a348599212cc1960f2dd6e7433d064faad762ecb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000005_20260801T144510252601Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7570b91bfb18b45bd8e2b95421f9f9cc517e96b550e75fbea4b2a743ace1c16e +size 6622417 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T144511613757Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T144511613757Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d65ac1ffb52fefe20cb76fe527c98ed5ac4a1903 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T144511613757Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116360536576, + "active_bytes_peak_delta": 2187776, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116360536576, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655963136, + "device": 1, + "inactive_split_bytes_current": 10162572288, + "inactive_split_bytes_peak": 15766232064, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:11.613247Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115655963136, + "allocated_bytes_peak_delta": 2257408, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116358348800, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115653705728, + "device": 1, + "inactive_split_bytes_current": 7690030080, + "inactive_split_bytes_peak": 15766232064, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:10.376546Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10162572288, + "inactive_split_bytes_current_delta": 2472542208, + "inactive_split_bytes_peak": 15766232064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122222018560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122222018560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000006_20260801T144511613757Z.pickle", + "timestamp_utc": "2026-08-01T14:45:11.722993Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T144511613757Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T144511613757Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5c425a14129b25f48787ef41684e4f2f3652212b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000006_20260801T144511613757Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d02465be9cccaa9b435a4472315f60a83607956550f67a8b950a21980d39640c +size 6640784 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T144513084279Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T144513084279Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d9c4dc12f6a1beefe3248529242d9a729a6b3c25 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T144513084279Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116360536576, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116360536576, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115658846208, + "device": 1, + "inactive_split_bytes_current": 10219195392, + "inactive_split_bytes_peak": 15766232064, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:13.083797Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115658846208, + "allocated_bytes_peak_delta": 2883072, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116360536576, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655963136, + "device": 1, + "inactive_split_bytes_current": 10162572288, + "inactive_split_bytes_peak": 15766232064, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:11.740595Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 10219195392, + "inactive_split_bytes_current_delta": 56623104, + "inactive_split_bytes_peak": 15766232064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122222018560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122222018560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000007_20260801T144513084279Z.pickle", + "timestamp_utc": "2026-08-01T14:45:13.194325Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T144513084279Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T144513084279Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a91ecfadca1dfa894784bd8eaa53455c727be9e4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000007_20260801T144513084279Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d02222b2a5c000a30618657a3360d96dc21500c1a4fcf5cdbb755562a0d2c81 +size 6622634 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T144514411860Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T144514411860Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6a5934b9a35d10dc3ff99016e0b072b3b0bd1802 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T144514411860Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116365910016, + "active_bytes_peak_delta": 5373440, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116365910016, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115661266944, + "device": 1, + "inactive_split_bytes_current": 8044448768, + "inactive_split_bytes_peak": 15766232064, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:14.411462Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115661266944, + "allocated_bytes_peak_delta": 2420736, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116360536576, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115658846208, + "device": 1, + "inactive_split_bytes_current": 10219195392, + "inactive_split_bytes_peak": 15766232064, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:13.209024Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 8044448768, + "inactive_split_bytes_current_delta": -2174746624, + "inactive_split_bytes_peak": 15766232064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122222018560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122222018560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000008_20260801T144514411860Z.pickle", + "timestamp_utc": "2026-08-01T14:45:14.517697Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T144514411860Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T144514411860Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..81de76ff2719400f2c09a2e57db3493aa83da481 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000008_20260801T144514411860Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9ce634bc897dca8e71375d6fd5c6c623c0accf29477e439fbb56828c10b0fb0 +size 6640413 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T144614834126Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T144614834126Z.json new file mode 100644 index 0000000000000000000000000000000000000000..76b09af07121d02a8e561d587ea8e327c13ddbdb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T144614834126Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 116365910016, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 116365910016, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 115661266944, + "device": 1, + "inactive_split_bytes_current": 11093707776, + "inactive_split_bytes_peak": 15766232064, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:46:14.833666Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 115661266944, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116365910016, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115661266944, + "device": 1, + "inactive_split_bytes_current": 8044448768, + "inactive_split_bytes_peak": 15766232064, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 1, + "reserved_bytes_current": 122222018560, + "reserved_bytes_peak": 122222018560, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:14.531692Z", + "world_size": 8 + }, + "device": 1, + "device_delta": 0, + "inactive_split_bytes_current": 11093707776, + "inactive_split_bytes_current_delta": 3049259008, + "inactive_split_bytes_peak": 15766232064, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 1, + "rank_delta": 0, + "reserved_bytes_current": 122222018560, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122222018560, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0001_iter000009_20260801T144614834126Z.pickle", + "timestamp_utc": "2026-08-01T14:46:15.047943Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T144614834126Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T144614834126Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2230d81978bc1d866983a8f3d6baed986841ec64 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0001_iter000009_20260801T144614834126Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b44639a52287022d071b5c7f0c0061ebf13eaa11b11bf0f96d0db50616caca84 +size 10542560 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T144503143596Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T144503143596Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d18e173c7e1ed1f457c6d590da41bcf1f366585b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T144503143596Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440796160, + "active_bytes_current_delta": 37209340928, + "active_bytes_peak": 93052798464, + "active_bytes_peak_delta": 57586462208, + "after": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 93052798464, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 91996358656, + "device": 2, + "inactive_split_bytes_current": 6282095616, + "inactive_split_bytes_peak": 12450221056, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 99828629504, + "reserved_bytes_peak": 99828629504, + "segment_count_current": 388, + "segment_count_peak": 388, + "timestamp_utc": "2026-08-01T14:45:03.142982Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440796160, + "allocated_bytes_current_delta": 37209340928, + "allocated_bytes_peak": 91996358656, + "allocated_bytes_peak_delta": 56530022400, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 2, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:43:54.202188Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 6282095616, + "inactive_split_bytes_current_delta": 6149276672, + "inactive_split_bytes_peak": 12450221056, + "inactive_split_bytes_peak_delta": 12289835008, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 99828629504, + "reserved_bytes_current_delta": 60584624128, + "reserved_bytes_peak": 99828629504, + "reserved_bytes_peak_delta": 60584624128, + "segment_count_current": 388, + "segment_count_current_delta": 317, + "segment_count_peak": 388, + "segment_count_peak_delta": 317, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000000_20260801T144503143596Z.pickle", + "timestamp_utc": "2026-08-01T14:45:03.260635Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T144503143596Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T144503143596Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27b1c40f8e5451f2122e176eb6fa59b6353c4c6d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000000_20260801T144503143596Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49f5473220744f67401d6a0d2493a9f469ccce782657ac39c2e373d12a7f799e +size 5945952 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T144504559811Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T144504559811Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3d661172dc51ce318863fdb1c6ddc8fb9744fa7d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T144504559811Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440796160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116348111360, + "active_bytes_peak_delta": 23295312896, + "after": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116348111360, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115649775616, + "device": 2, + "inactive_split_bytes_current": 9278925824, + "inactive_split_bytes_peak": 14353191936, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 121043419136, + "reserved_bytes_peak": 121043419136, + "segment_count_current": 439, + "segment_count_peak": 439, + "timestamp_utc": "2026-08-01T14:45:04.559395Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440796160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115649775616, + "allocated_bytes_peak_delta": 23653416960, + "before": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 93052798464, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 91996358656, + "device": 2, + "inactive_split_bytes_current": 6282095616, + "inactive_split_bytes_peak": 12450221056, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 99828629504, + "reserved_bytes_peak": 99828629504, + "segment_count_current": 388, + "segment_count_peak": 388, + "timestamp_utc": "2026-08-01T14:45:03.278941Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9278925824, + "inactive_split_bytes_current_delta": 2996830208, + "inactive_split_bytes_peak": 14353191936, + "inactive_split_bytes_peak_delta": 1902970880, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 121043419136, + "reserved_bytes_current_delta": 21214789632, + "reserved_bytes_peak": 121043419136, + "reserved_bytes_peak_delta": 21214789632, + "segment_count_current": 439, + "segment_count_current_delta": 51, + "segment_count_peak": 439, + "segment_count_peak_delta": 51, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000001_20260801T144504559811Z.pickle", + "timestamp_utc": "2026-08-01T14:45:04.677088Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T144504559811Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T144504559811Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4791fc6db6a43cd0662fa596f2649e98f16a4ffb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000001_20260801T144504559811Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8626675e2ee86f2ffca65bea977609c12bc626bcf55051b5fe93266207406eae +size 5959403 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T144505924421Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T144505924421Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0680ef3ec15bc588683d8519ba5078b9afe65474 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T144505924421Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440796160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707043328, + "active_bytes_peak_delta": 358931968, + "after": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115650078720, + "device": 2, + "inactive_split_bytes_current": 8454745088, + "inactive_split_bytes_peak": 15178585088, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:05.924014Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440796160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115650078720, + "allocated_bytes_peak_delta": 303104, + "before": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116348111360, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115649775616, + "device": 2, + "inactive_split_bytes_current": 9278925824, + "inactive_split_bytes_peak": 14353191936, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 121043419136, + "reserved_bytes_peak": 121043419136, + "segment_count_current": 439, + "segment_count_peak": 439, + "timestamp_utc": "2026-08-01T14:45:04.695101Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8454745088, + "inactive_split_bytes_current_delta": -824180736, + "inactive_split_bytes_peak": 15178585088, + "inactive_split_bytes_peak_delta": 825393152, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 121867599872, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 121867599872, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 440, + "segment_count_current_delta": 1, + "segment_count_peak": 440, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000002_20260801T144505924421Z.pickle", + "timestamp_utc": "2026-08-01T14:45:06.033302Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T144505924421Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T144505924421Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..3c66866abfb558c058f88b9eafd94b836e405e8f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000002_20260801T144505924421Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b24a90cbc26d956d67c260a234bd0ed720401a3faf783e667f642a1afd6b6d36 +size 6119630 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T144507273701Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T144507273701Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f92198afc032b9e0a79648712589e8cf302b9feb --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T144507273701Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440796160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707043328, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115659597824, + "device": 2, + "inactive_split_bytes_current": 9513806848, + "inactive_split_bytes_peak": 15178585088, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:07.273288Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440796160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115659597824, + "allocated_bytes_peak_delta": 9519104, + "before": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115650078720, + "device": 2, + "inactive_split_bytes_current": 8454745088, + "inactive_split_bytes_peak": 15178585088, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:06.048857Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9513806848, + "inactive_split_bytes_current_delta": 1059061760, + "inactive_split_bytes_peak": 15178585088, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 121867599872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121867599872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 440, + "segment_count_current_delta": 0, + "segment_count_peak": 440, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000003_20260801T144507273701Z.pickle", + "timestamp_utc": "2026-08-01T14:45:07.383270Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T144507273701Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T144507273701Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..34196a5d00d315f13f48bec2b64396f2745cae38 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000003_20260801T144507273701Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df307c94ce8e87ca65b5c69099cd3c4892e9a370da26f9d7a3d655de0ffd0c77 +size 6623420 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T144508911487Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T144508911487Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0bdfd6a2d865ca8cbcb00dd861ce66472ddc3e91 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T144508911487Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440796160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707043328, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115659597824, + "device": 2, + "inactive_split_bytes_current": 9629150208, + "inactive_split_bytes_peak": 15357367296, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:08.911054Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440796160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115659597824, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115659597824, + "device": 2, + "inactive_split_bytes_current": 9513806848, + "inactive_split_bytes_peak": 15178585088, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:07.398708Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9629150208, + "inactive_split_bytes_current_delta": 115343360, + "inactive_split_bytes_peak": 15357367296, + "inactive_split_bytes_peak_delta": 178782208, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 121867599872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121867599872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 440, + "segment_count_current_delta": 0, + "segment_count_peak": 440, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000004_20260801T144508911487Z.pickle", + "timestamp_utc": "2026-08-01T14:45:09.023391Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T144508911487Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T144508911487Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..077bea5c3e15d0019de208499f4c069a3781612a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000004_20260801T144508911487Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29edcca67106ba48dffbfb148274ea3b21e12a3581eaa0e3e51e735941aa3869 +size 6623334 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T144510252598Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T144510252598Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b5926ae77cb8f2d4040bf48c28f168a72119dcbd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T144510252598Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440796160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707043328, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115659597824, + "device": 2, + "inactive_split_bytes_current": 9631247360, + "inactive_split_bytes_peak": 15357367296, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:10.252189Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440796160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115659597824, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115659597824, + "device": 2, + "inactive_split_bytes_current": 9629150208, + "inactive_split_bytes_peak": 15357367296, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:09.038922Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9631247360, + "inactive_split_bytes_current_delta": 2097152, + "inactive_split_bytes_peak": 15357367296, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 121867599872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121867599872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 440, + "segment_count_current_delta": 0, + "segment_count_peak": 440, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000005_20260801T144510252598Z.pickle", + "timestamp_utc": "2026-08-01T14:45:10.361022Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T144510252598Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T144510252598Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..64e01ac1e6982aeffc4bea7d8b5fc416b6db3534 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000005_20260801T144510252598Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ab46d0bad1b05e99a4e35a9a0605d760ff63d361d90c0e6a777b73e2aaf9e5e +size 6623292 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T144511613552Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T144511613552Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f76e249742852c8cfaa4dee82bf39d5764a605e2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T144511613552Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440796160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707043328, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115659597824, + "device": 2, + "inactive_split_bytes_current": 8983227392, + "inactive_split_bytes_peak": 15757358080, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 122691780608, + "reserved_bytes_peak": 122691780608, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:11.613155Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440796160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115659597824, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115659597824, + "device": 2, + "inactive_split_bytes_current": 9631247360, + "inactive_split_bytes_peak": 15357367296, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:10.376600Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 8983227392, + "inactive_split_bytes_current_delta": -648019968, + "inactive_split_bytes_peak": 15757358080, + "inactive_split_bytes_peak_delta": 399990784, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 122691780608, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 122691780608, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 441, + "segment_count_current_delta": 1, + "segment_count_peak": 441, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000006_20260801T144511613552Z.pickle", + "timestamp_utc": "2026-08-01T14:45:11.719927Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T144511613552Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T144511613552Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4c2e612efc449f9d77e96db1f29bc322638de3ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000006_20260801T144511613552Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d0168dd9549226cfa256cd4abbe9186c36a19d8db51929cba54b040b20cea75 +size 6117432 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T144513084121Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T144513084121Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c6bf617870329ba6ea439563adeb17e4b66f7fe2 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T144513084121Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440796160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707043328, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115659597824, + "device": 2, + "inactive_split_bytes_current": 9337646080, + "inactive_split_bytes_peak": 15939126272, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 122691780608, + "reserved_bytes_peak": 122691780608, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:13.083707Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440796160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115659597824, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115659597824, + "device": 2, + "inactive_split_bytes_current": 8983227392, + "inactive_split_bytes_peak": 15757358080, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 122691780608, + "reserved_bytes_peak": 122691780608, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:11.734945Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9337646080, + "inactive_split_bytes_current_delta": 354418688, + "inactive_split_bytes_peak": 15939126272, + "inactive_split_bytes_peak_delta": 181768192, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 122691780608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122691780608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000007_20260801T144513084121Z.pickle", + "timestamp_utc": "2026-08-01T14:45:13.191222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T144513084121Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T144513084121Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6416c8ebd0f190e725792b5f2dc151ee10a664da --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000007_20260801T144513084121Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1ab8d106d58fb916bac0e8520dc0e1081ba86a84e0359807a4e8d9741a26a7d +size 6623207 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T144514411881Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T144514411881Z.json new file mode 100644 index 0000000000000000000000000000000000000000..46f576fb3874fbe5a475d016801ae83229f00006 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T144514411881Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440796160, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116707043328, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115663065088, + "device": 2, + "inactive_split_bytes_current": 9985666048, + "inactive_split_bytes_peak": 16172436480, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 122691780608, + "reserved_bytes_peak": 122691780608, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:14.411478Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440796160, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115663065088, + "allocated_bytes_peak_delta": 3467264, + "before": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115659597824, + "device": 2, + "inactive_split_bytes_current": 9337646080, + "inactive_split_bytes_peak": 15939126272, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 122691780608, + "reserved_bytes_peak": 122691780608, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:13.206496Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 9985666048, + "inactive_split_bytes_current_delta": 648019968, + "inactive_split_bytes_peak": 16172436480, + "inactive_split_bytes_peak_delta": 233310208, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 122691780608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122691780608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000008_20260801T144514411881Z.pickle", + "timestamp_utc": "2026-08-01T14:45:14.525016Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T144514411881Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T144514411881Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27236d86b61aa8a1c0709d6a419d8be577df557c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000008_20260801T144514411881Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fc966ff3c027b3365369cb34d7a0ef10c98770f2b29211e73b6e3e95dae53eb +size 6623319 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T144614834127Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T144614834127Z.json new file mode 100644 index 0000000000000000000000000000000000000000..910db1d38c763b125e77cc42407340c9389f263e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T144614834127Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61166507008, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 116707043328, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61166507008, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 61166507008, + "allocated_bytes_peak": 115663065088, + "device": 2, + "inactive_split_bytes_current": 10623200256, + "inactive_split_bytes_peak": 16172436480, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 2, + "reserved_bytes_current": 122691780608, + "reserved_bytes_peak": 122691780608, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:46:14.833658Z", + "world_size": 8 + }, + "allocated_bytes_current": 61166507008, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 115663065088, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440796160, + "active_bytes_peak": 116707043328, + "allocated_bytes_current": 72440796160, + "allocated_bytes_peak": 115663065088, + "device": 2, + "inactive_split_bytes_current": 9985666048, + "inactive_split_bytes_peak": 16172436480, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 2, + "reserved_bytes_current": 122691780608, + "reserved_bytes_peak": 122691780608, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:14.542823Z", + "world_size": 8 + }, + "device": 2, + "device_delta": 0, + "inactive_split_bytes_current": 10623200256, + "inactive_split_bytes_current_delta": 637534208, + "inactive_split_bytes_peak": 16172436480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 2, + "rank_delta": 0, + "reserved_bytes_current": 122691780608, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122691780608, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0002_iter000009_20260801T144614834127Z.pickle", + "timestamp_utc": "2026-08-01T14:46:15.038490Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T144614834127Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T144614834127Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..58ced0e2e63d5d5fc63cead3150e9a692ba363b7 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0002_iter000009_20260801T144614834127Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5be7645b45953c2216cb5344235716338aa2dc03ef7732e67adcfe743e10c1af +size 10543013 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T144503143751Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T144503143751Z.json new file mode 100644 index 0000000000000000000000000000000000000000..91619c32a5a2caee935095fb45f7602eaa8163a1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T144503143751Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 92695639552, + "active_bytes_peak_delta": 57229303296, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92695639552, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 91993606144, + "device": 3, + "inactive_split_bytes_current": 6813420544, + "inactive_split_bytes_peak": 11939119616, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 99375644672, + "reserved_bytes_peak": 99375644672, + "segment_count_current": 386, + "segment_count_peak": 386, + "timestamp_utc": "2026-08-01T14:45:03.143086Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 91993606144, + "allocated_bytes_peak_delta": 56527269888, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 3, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:43:54.202796Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 6813420544, + "inactive_split_bytes_current_delta": 6680601600, + "inactive_split_bytes_peak": 11939119616, + "inactive_split_bytes_peak_delta": 11778733568, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 99375644672, + "reserved_bytes_current_delta": 60131639296, + "reserved_bytes_peak": 99375644672, + "reserved_bytes_peak_delta": 60131639296, + "segment_count_current": 386, + "segment_count_current_delta": 315, + "segment_count_peak": 386, + "segment_count_peak_delta": 315, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000000_20260801T144503143751Z.pickle", + "timestamp_utc": "2026-08-01T14:45:03.286378Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T144503143751Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T144503143751Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7edd95868f201a5e89b8033a051a7989801b9430 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000000_20260801T144503143751Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3924d212c3666bfd8d8834a8848af88a101db23e1d9b8d083a28f3fef330febc +size 5945859 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T144504560063Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T144504560063Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f4f6e19ae402cefcd6745ac244568c476fd10e6e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T144504560063Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116000586240, + "active_bytes_peak_delta": 23304946688, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116000586240, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115659634688, + "device": 3, + "inactive_split_bytes_current": 8457587712, + "inactive_split_bytes_peak": 14702228480, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:04.559555Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115659634688, + "allocated_bytes_peak_delta": 23666028544, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92695639552, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 91993606144, + "device": 3, + "inactive_split_bytes_current": 6813420544, + "inactive_split_bytes_peak": 11939119616, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 99375644672, + "reserved_bytes_peak": 99375644672, + "segment_count_current": 386, + "segment_count_peak": 386, + "timestamp_utc": "2026-08-01T14:45:03.316948Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8457587712, + "inactive_split_bytes_current_delta": 1644167168, + "inactive_split_bytes_peak": 14702228480, + "inactive_split_bytes_peak_delta": 2763108864, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122393985024, + "reserved_bytes_current_delta": 23018340352, + "reserved_bytes_peak": 122393985024, + "reserved_bytes_peak_delta": 23018340352, + "segment_count_current": 443, + "segment_count_current_delta": 57, + "segment_count_peak": 443, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000001_20260801T144504560063Z.pickle", + "timestamp_utc": "2026-08-01T14:45:04.681978Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T144504560063Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T144504560063Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..228ae0fa0738e690cab13cdd45b9d936ae5f23c0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000001_20260801T144504560063Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d33c412c91e113a3027ca70bd5a8a97ce0160d635f7a79245167a1998c6a7f7c +size 5947384 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T144505924695Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T144505924695Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f1645796713c29abf1cec52d66a551d8d28163ab --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T144505924695Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116361394176, + "active_bytes_peak_delta": 360807936, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115659634688, + "device": 3, + "inactive_split_bytes_current": 9512455168, + "inactive_split_bytes_peak": 14702228480, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:05.924180Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115659634688, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116000586240, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115659634688, + "device": 3, + "inactive_split_bytes_current": 8457587712, + "inactive_split_bytes_peak": 14702228480, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:04.698218Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9512455168, + "inactive_split_bytes_current_delta": 1054867456, + "inactive_split_bytes_peak": 14702228480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122393985024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122393985024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 443, + "segment_count_current_delta": 0, + "segment_count_peak": 443, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000002_20260801T144505924695Z.pickle", + "timestamp_utc": "2026-08-01T14:45:06.035645Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T144505924695Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T144505924695Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..51a5cc62e8d5cc257e6f9a14e325efce36792f3c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000002_20260801T144505924695Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91039a6e0e8048f36eb3c57c11bebb4afbdfe8353ca280811eff80e1f46734a8 +size 6624305 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T144507273893Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T144507273893Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7ff1fea52276c994b4852861863fb9af11f99807 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T144507273893Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116361394176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115659634688, + "device": 3, + "inactive_split_bytes_current": 9160133632, + "inactive_split_bytes_peak": 14702228480, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:07.273387Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115659634688, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115659634688, + "device": 3, + "inactive_split_bytes_current": 9512455168, + "inactive_split_bytes_peak": 14702228480, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:06.051089Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9160133632, + "inactive_split_bytes_current_delta": -352321536, + "inactive_split_bytes_peak": 14702228480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122393985024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122393985024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 443, + "segment_count_current_delta": 0, + "segment_count_peak": 443, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000003_20260801T144507273893Z.pickle", + "timestamp_utc": "2026-08-01T14:45:07.388991Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T144507273893Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T144507273893Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..613c63fc1f60076dc9126f80408d2b6d3b32d6f9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000003_20260801T144507273893Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77fe78a3606e17b738ebfbca069cb3a81442070072b17df1825690af826473b5 +size 6623324 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T144508911728Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T144508911728Z.json new file mode 100644 index 0000000000000000000000000000000000000000..ed5984be91f28ecc1921cfa4ce5c24a6f1ef0df0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T144508911728Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116361394176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115659634688, + "device": 3, + "inactive_split_bytes_current": 9275476992, + "inactive_split_bytes_peak": 14702228480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:08.911209Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115659634688, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115659634688, + "device": 3, + "inactive_split_bytes_current": 9160133632, + "inactive_split_bytes_peak": 14702228480, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:07.404749Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9275476992, + "inactive_split_bytes_current_delta": 115343360, + "inactive_split_bytes_peak": 14702228480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122393985024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122393985024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 443, + "segment_count_current_delta": 0, + "segment_count_peak": 443, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000004_20260801T144508911728Z.pickle", + "timestamp_utc": "2026-08-01T14:45:09.026612Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T144508911728Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T144508911728Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..84ed63a459d5f6f0175988c37c93e937210724b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000004_20260801T144508911728Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccdc7a4dc21666e1f25dfac898daad37771b3ee2ef7bc50656dc58477ce4d296 +size 6623387 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T144510252970Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T144510252970Z.json new file mode 100644 index 0000000000000000000000000000000000000000..faecae527c4efbae97cd87a6dee7be4db777efb3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T144510252970Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116361394176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115659634688, + "device": 3, + "inactive_split_bytes_current": 10158377984, + "inactive_split_bytes_peak": 14702228480, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:10.252397Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115659634688, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115659634688, + "device": 3, + "inactive_split_bytes_current": 9275476992, + "inactive_split_bytes_peak": 14702228480, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:09.042453Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 10158377984, + "inactive_split_bytes_current_delta": 882900992, + "inactive_split_bytes_peak": 14702228480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122393985024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122393985024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 443, + "segment_count_current_delta": 0, + "segment_count_peak": 443, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000005_20260801T144510252970Z.pickle", + "timestamp_utc": "2026-08-01T14:45:10.368751Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T144510252970Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T144510252970Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4729e54ac2bdfc876552570766409a8aa984b8f1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000005_20260801T144510252970Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:745658d433d6f3396849bd3b3883a0953cf2ee7e0ba150b4c3918cf0e953062d +size 6641414 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T144511613821Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T144511613821Z.json new file mode 100644 index 0000000000000000000000000000000000000000..18758ecb62e720e3b1b1f1a864df74e99ca1d43c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T144511613821Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116361394176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115660158976, + "device": 3, + "inactive_split_bytes_current": 11045473280, + "inactive_split_bytes_peak": 14702228480, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:11.613316Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115660158976, + "allocated_bytes_peak_delta": 524288, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115659634688, + "device": 3, + "inactive_split_bytes_current": 10158377984, + "inactive_split_bytes_peak": 14702228480, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:10.384762Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 11045473280, + "inactive_split_bytes_current_delta": 887095296, + "inactive_split_bytes_peak": 14702228480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122393985024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122393985024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 443, + "segment_count_current_delta": 0, + "segment_count_peak": 443, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000006_20260801T144511613821Z.pickle", + "timestamp_utc": "2026-08-01T14:45:11.733844Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T144511613821Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T144511613821Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..032af05ab33184f511a75f3c2ed7350e18d6867f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000006_20260801T144511613821Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e03773caab2f7e3bce0db875f89b8e84c4d9e91c07232128a0943fe06048c5ca +size 6622929 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T144513084324Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T144513084324Z.json new file mode 100644 index 0000000000000000000000000000000000000000..db2f144086ee599605eea94063c92bb26495f77c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T144513084324Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116361394176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115660158976, + "device": 3, + "inactive_split_bytes_current": 8631651328, + "inactive_split_bytes_peak": 14702228480, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:13.083813Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115660158976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115660158976, + "device": 3, + "inactive_split_bytes_current": 11045473280, + "inactive_split_bytes_peak": 14702228480, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:11.753075Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 8631651328, + "inactive_split_bytes_current_delta": -2413821952, + "inactive_split_bytes_peak": 14702228480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122393985024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122393985024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 443, + "segment_count_current_delta": 0, + "segment_count_peak": 443, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000007_20260801T144513084324Z.pickle", + "timestamp_utc": "2026-08-01T14:45:13.194370Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T144513084324Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T144513084324Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ac3370053a003e37f0eca64a90f2aea019ded9ca --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000007_20260801T144513084324Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4e5c97c7df1d25e397d029122e9ea37964098e4f809346b50985ce5ad4040bf +size 6622643 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T144514412119Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T144514412119Z.json new file mode 100644 index 0000000000000000000000000000000000000000..173eaaa49361630b923926983bc5aebd072d7621 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T144514412119Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116361394176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115660158976, + "device": 3, + "inactive_split_bytes_current": 9514552320, + "inactive_split_bytes_peak": 14702228480, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:14.411595Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115660158976, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115660158976, + "device": 3, + "inactive_split_bytes_current": 8631651328, + "inactive_split_bytes_peak": 14702228480, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:13.208849Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9514552320, + "inactive_split_bytes_current_delta": 882900992, + "inactive_split_bytes_peak": 14702228480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122393985024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122393985024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 443, + "segment_count_current_delta": 0, + "segment_count_peak": 443, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000008_20260801T144514412119Z.pickle", + "timestamp_utc": "2026-08-01T14:45:14.522426Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T144514412119Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T144514412119Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ea8460634704a2571cfbc3f1edb2e2e02350acba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000008_20260801T144514412119Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c8dce5147e703c2c7d47c2d609cc8f9952225485b44e9876c7728d280ea2efb +size 6622701 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T144614834181Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T144614834181Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0d5d56731b5db9fc4b52b5e72718b827c817b818 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T144614834181Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 116361394176, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 115662717952, + "device": 3, + "inactive_split_bytes_current": 9625701376, + "inactive_split_bytes_peak": 14702228480, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:46:14.833683Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 115662717952, + "allocated_bytes_peak_delta": 2558976, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116361394176, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115660158976, + "device": 3, + "inactive_split_bytes_current": 9514552320, + "inactive_split_bytes_peak": 14702228480, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 3, + "reserved_bytes_current": 122393985024, + "reserved_bytes_peak": 122393985024, + "segment_count_current": 443, + "segment_count_peak": 443, + "timestamp_utc": "2026-08-01T14:45:14.537019Z", + "world_size": 8 + }, + "device": 3, + "device_delta": 0, + "inactive_split_bytes_current": 9625701376, + "inactive_split_bytes_current_delta": 111149056, + "inactive_split_bytes_peak": 14702228480, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 3, + "rank_delta": 0, + "reserved_bytes_current": 122393985024, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122393985024, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 443, + "segment_count_current_delta": 0, + "segment_count_peak": 443, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0003_iter000009_20260801T144614834181Z.pickle", + "timestamp_utc": "2026-08-01T14:46:15.054647Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T144614834181Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T144614834181Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..f5419abe4e6179a04e453e446a9f87a9a6295410 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0003_iter000009_20260801T144614834181Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b96f167f660b3e479f40500fc99e14db8dfa406596e8853c4d98392e92687e28 +size 10542457 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T144503143634Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T144503143634Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8b16a4905a574feeb0391c66cd53e20036922cd4 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T144503143634Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441705472, + "active_bytes_current_delta": 37209807872, + "active_bytes_peak": 92705174016, + "active_bytes_peak_delta": 57238395392, + "after": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 92705174016, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 92000793600, + "device": 4, + "inactive_split_bytes_current": 6356683776, + "inactive_split_bytes_peak": 12619963904, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 98905882624, + "reserved_bytes_peak": 98905882624, + "segment_count_current": 380, + "segment_count_peak": 380, + "timestamp_utc": "2026-08-01T14:45:03.143057Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441705472, + "allocated_bytes_current_delta": 37209807872, + "allocated_bytes_peak": 92000793600, + "allocated_bytes_peak_delta": 56534014976, + "before": { + "active_bytes_current": 35231897600, + "active_bytes_peak": 35466778624, + "allocated_bytes_current": 35231897600, + "allocated_bytes_peak": 35466778624, + "device": 4, + "inactive_split_bytes_current": 132376576, + "inactive_split_bytes_peak": 153409536, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:43:54.202824Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 6356683776, + "inactive_split_bytes_current_delta": 6224307200, + "inactive_split_bytes_peak": 12619963904, + "inactive_split_bytes_peak_delta": 12466554368, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 98905882624, + "reserved_bytes_current_delta": 59661877248, + "reserved_bytes_peak": 98905882624, + "reserved_bytes_peak_delta": 59661877248, + "segment_count_current": 380, + "segment_count_current_delta": 309, + "segment_count_peak": 380, + "segment_count_peak_delta": 309, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000000_20260801T144503143634Z.pickle", + "timestamp_utc": "2026-08-01T14:45:03.260024Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T144503143634Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T144503143634Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8f677d9d2705b9dcef7ff8f50bd4c1a9ff6ce7ff --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000000_20260801T144503143634Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8ee93c53bc593955ac92913c12ebfe1df835118b8b40a47639368c951f577a0 +size 5953570 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T144504559747Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T144504559747Z.json new file mode 100644 index 0000000000000000000000000000000000000000..06a28a3fa32a797a40f7e4a12b2b26bcc0783e31 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T144504559747Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441705472, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116344889856, + "active_bytes_peak_delta": 23639715840, + "after": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116344889856, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115649531904, + "device": 4, + "inactive_split_bytes_current": 8002948096, + "inactive_split_bytes_peak": 14770977792, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:04.559351Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441705472, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115649531904, + "allocated_bytes_peak_delta": 23648738304, + "before": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 92705174016, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 92000793600, + "device": 4, + "inactive_split_bytes_current": 6356683776, + "inactive_split_bytes_peak": 12619963904, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 98905882624, + "reserved_bytes_peak": 98905882624, + "segment_count_current": 380, + "segment_count_peak": 380, + "timestamp_utc": "2026-08-01T14:45:03.277438Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8002948096, + "inactive_split_bytes_current_delta": 1646264320, + "inactive_split_bytes_peak": 14770977792, + "inactive_split_bytes_peak_delta": 2151013888, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121221677056, + "reserved_bytes_current_delta": 22315794432, + "reserved_bytes_peak": 121221677056, + "reserved_bytes_peak_delta": 22315794432, + "segment_count_current": 437, + "segment_count_current_delta": 57, + "segment_count_peak": 437, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000001_20260801T144504559747Z.pickle", + "timestamp_utc": "2026-08-01T14:45:04.672965Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T144504559747Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T144504559747Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a4bbd8c25aa332f692f8a0b36e2ffcdc7bd2f552 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000001_20260801T144504559747Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3185921a4259dc4c63094c7afd1efb9e9e38478651c1f1e7a6c4a32f915a2f13 +size 6113108 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T144505924365Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T144505924365Z.json new file mode 100644 index 0000000000000000000000000000000000000000..2a88cce65d2206822f4101cae9c83c393788d830 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T144505924365Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441705472, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116715671552, + "active_bytes_peak_delta": 370781696, + "after": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115658706944, + "device": 4, + "inactive_split_bytes_current": 9236073472, + "inactive_split_bytes_peak": 14770977792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:05.923965Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441705472, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115658706944, + "allocated_bytes_peak_delta": 9175040, + "before": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116344889856, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115649531904, + "device": 4, + "inactive_split_bytes_current": 8002948096, + "inactive_split_bytes_peak": 14770977792, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:04.685553Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9236073472, + "inactive_split_bytes_current_delta": 1233125376, + "inactive_split_bytes_peak": 14770977792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121221677056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121221677056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 437, + "segment_count_current_delta": 0, + "segment_count_peak": 437, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000002_20260801T144505924365Z.pickle", + "timestamp_utc": "2026-08-01T14:45:06.025621Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T144505924365Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T144505924365Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4bdde3c318525964b3ab127aeff461bdd01ef091 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000002_20260801T144505924365Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f04fb662cf8d9fb3425e6023e5a3a680581dbe1150da6cd24840afb0f72c7e9 +size 6630325 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T144507273638Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T144507273638Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9c2876f08a521c5d945722b180e6f0b8100f3fc5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T144507273638Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441705472, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116715671552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 9233976320, + "inactive_split_bytes_peak": 14770977792, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:07.273241Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441705472, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115662512128, + "allocated_bytes_peak_delta": 3805184, + "before": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115658706944, + "device": 4, + "inactive_split_bytes_current": 9236073472, + "inactive_split_bytes_peak": 14770977792, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:06.037486Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9233976320, + "inactive_split_bytes_current_delta": -2097152, + "inactive_split_bytes_peak": 14770977792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121221677056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121221677056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 437, + "segment_count_current_delta": 0, + "segment_count_peak": 437, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000003_20260801T144507273638Z.pickle", + "timestamp_utc": "2026-08-01T14:45:07.378862Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T144507273638Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T144507273638Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..20095bc3aa1154af4ac507606b3fdb282ce2ff65 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000003_20260801T144507273638Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5e0099cd3e17b398243f7dabb335f33c7ea0e8456a00d0f1028ee5b5480e228 +size 6629653 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T144508911444Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T144508911444Z.json new file mode 100644 index 0000000000000000000000000000000000000000..04c97f7697712d8c086abf27d6d6501e321e8b83 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T144508911444Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441705472, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116715671552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 8057474048, + "inactive_split_bytes_peak": 14770977792, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:08.911029Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441705472, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115662512128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 9233976320, + "inactive_split_bytes_peak": 14770977792, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:07.391125Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8057474048, + "inactive_split_bytes_current_delta": -1176502272, + "inactive_split_bytes_peak": 14770977792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121221677056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121221677056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 437, + "segment_count_current_delta": 0, + "segment_count_peak": 437, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000004_20260801T144508911444Z.pickle", + "timestamp_utc": "2026-08-01T14:45:09.018884Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T144508911444Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T144508911444Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1a9abb34a0a3278ac7d1eda4c9a3f5278dcb3a5a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000004_20260801T144508911444Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a78e5c71b691b9e1abc23baab67fad79264ce946e6cff1da740ba52e5702b18 +size 6629175 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T144510252567Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T144510252567Z.json new file mode 100644 index 0000000000000000000000000000000000000000..36191fc7d3fe2b3ae6fbb3b5d33b831e192c92cc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T144510252567Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441705472, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116715671552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 8470612992, + "inactive_split_bytes_peak": 14770977792, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:10.252156Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441705472, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115662512128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 8057474048, + "inactive_split_bytes_peak": 14770977792, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:09.031312Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8470612992, + "inactive_split_bytes_current_delta": 413138944, + "inactive_split_bytes_peak": 14770977792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121221677056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121221677056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 437, + "segment_count_current_delta": 0, + "segment_count_peak": 437, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000005_20260801T144510252567Z.pickle", + "timestamp_utc": "2026-08-01T14:45:10.355385Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T144510252567Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T144510252567Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..150fd45ed88508979184c30c5a7e0ee05aa25545 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000005_20260801T144510252567Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38031dfe69a972979eb94cd449c3caff07b04265b76fe5aa413fb40c7056ba66 +size 6629380 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T144511613579Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T144511613579Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b390444f683d3076012a12e5efc278d6d7ff8792 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T144511613579Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441705472, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116715671552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 7765969920, + "inactive_split_bytes_peak": 14770977792, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:11.613168Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441705472, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115662512128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 8470612992, + "inactive_split_bytes_peak": 14770977792, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:10.367392Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7765969920, + "inactive_split_bytes_current_delta": -704643072, + "inactive_split_bytes_peak": 14770977792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121221677056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121221677056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 437, + "segment_count_current_delta": 0, + "segment_count_peak": 437, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000006_20260801T144511613579Z.pickle", + "timestamp_utc": "2026-08-01T14:45:11.716250Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T144511613579Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T144511613579Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..ec384b3ec7b7e9ea562bc1f86b5319797f215795 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000006_20260801T144511613579Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3b023853cce930e706091ba7d23f064c2edde2770989dd4b845e706274398bd +size 6629184 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T144513084089Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T144513084089Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d4f68653f7806047917b44eec4cc1e68ec52ce0f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T144513084089Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441705472, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116715671552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 7587712000, + "inactive_split_bytes_peak": 14770977792, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:13.083691Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441705472, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115662512128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 7765969920, + "inactive_split_bytes_peak": 14770977792, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:11.728076Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 7587712000, + "inactive_split_bytes_current_delta": -178257920, + "inactive_split_bytes_peak": 14770977792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121221677056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121221677056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 437, + "segment_count_current_delta": 0, + "segment_count_peak": 437, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000007_20260801T144513084089Z.pickle", + "timestamp_utc": "2026-08-01T14:45:13.182934Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T144513084089Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T144513084089Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6c149b464210b5a4ecd0c0bf7b2536d70a18c2ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000007_20260801T144513084089Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:772f7f7a4ab8d434166e51ba00d21e43e12678c206fdec1ea766e43f3fb823d3 +size 6629144 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T144514411836Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T144514411836Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b3cc92bc7c1a85a8052e60062f580e5b9701b4db --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T144514411836Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441705472, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116715671552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 8529333248, + "inactive_split_bytes_peak": 14770977792, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:14.411434Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441705472, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115662512128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 7587712000, + "inactive_split_bytes_peak": 14770977792, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:13.194608Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 8529333248, + "inactive_split_bytes_current_delta": 941621248, + "inactive_split_bytes_peak": 14770977792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121221677056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121221677056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 437, + "segment_count_current_delta": 0, + "segment_count_peak": 437, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000008_20260801T144514411836Z.pickle", + "timestamp_utc": "2026-08-01T14:45:14.520218Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T144514411836Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T144514411836Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..286210ab7495c35451676b75094491de620948e9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000008_20260801T144514411836Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d84f5672fb2e8d670705f836982f6c030f3d34dc005a945015edde206ea56ef +size 6629212 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T144614834240Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T144614834240Z.json new file mode 100644 index 0000000000000000000000000000000000000000..05513584dc3e024154d542ae959284c5cb1b06c3 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T144614834240Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61167416320, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 116715671552, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61167416320, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 61167416320, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 9699544064, + "inactive_split_bytes_peak": 14770977792, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:46:14.833732Z", + "world_size": 8 + }, + "allocated_bytes_current": 61167416320, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 115662512128, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72441705472, + "active_bytes_peak": 116715671552, + "allocated_bytes_current": 72441705472, + "allocated_bytes_peak": 115662512128, + "device": 4, + "inactive_split_bytes_current": 8529333248, + "inactive_split_bytes_peak": 14770977792, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 4, + "reserved_bytes_current": 121221677056, + "reserved_bytes_peak": 121221677056, + "segment_count_current": 437, + "segment_count_peak": 437, + "timestamp_utc": "2026-08-01T14:45:14.533975Z", + "world_size": 8 + }, + "device": 4, + "device_delta": 0, + "inactive_split_bytes_current": 9699544064, + "inactive_split_bytes_current_delta": 1170210816, + "inactive_split_bytes_peak": 14770977792, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 4, + "rank_delta": 0, + "reserved_bytes_current": 121221677056, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121221677056, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 437, + "segment_count_current_delta": 0, + "segment_count_peak": 437, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0004_iter000009_20260801T144614834240Z.pickle", + "timestamp_utc": "2026-08-01T14:46:15.046582Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T144614834240Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T144614834240Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5960503d0c36a880625cb3ea8bd913abac7aa162 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0004_iter000009_20260801T144614834240Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55c3bbbd519b01f58224d4d09ffedbf6254563b03ddcc77923a3e5db12e4aa00 +size 10549300 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T144503143556Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T144503143556Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5d6ed6ec2f96264cf8b183dabceea4dc6ea3b8a8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T144503143556Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 93048604160, + "active_bytes_peak_delta": 57582267904, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 93048604160, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 91996227584, + "device": 5, + "inactive_split_bytes_current": 6045862912, + "inactive_split_bytes_peak": 12270207488, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 97479819264, + "reserved_bytes_peak": 97479819264, + "segment_count_current": 383, + "segment_count_peak": 383, + "timestamp_utc": "2026-08-01T14:45:03.142957Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 91996227584, + "allocated_bytes_peak_delta": 56529891328, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 5, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:43:54.202666Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 6045862912, + "inactive_split_bytes_current_delta": 5913043968, + "inactive_split_bytes_peak": 12270207488, + "inactive_split_bytes_peak_delta": 12109821440, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 97479819264, + "reserved_bytes_current_delta": 58235813888, + "reserved_bytes_peak": 97479819264, + "reserved_bytes_peak_delta": 58235813888, + "segment_count_current": 383, + "segment_count_current_delta": 312, + "segment_count_peak": 383, + "segment_count_peak_delta": 312, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000000_20260801T144503143556Z.pickle", + "timestamp_utc": "2026-08-01T14:45:03.260898Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T144503143556Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T144503143556Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..8329bc277ad0c6f313a75dcb0218814b19333842 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000000_20260801T144503143556Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa156e0a81f651eb38433bb0715b25380e370ad07d8b3f85148aadc0f42b148b +size 5945340 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T144504559763Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T144504559763Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d07e8b0286a59cd7b337197e0aacd74ca619677c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T144504559763Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116352543232, + "active_bytes_peak_delta": 23303939072, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116352543232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115651246080, + "device": 5, + "inactive_split_bytes_current": 8459684864, + "inactive_split_bytes_peak": 15290946560, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121043419136, + "reserved_bytes_peak": 121043419136, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:04.559356Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115651246080, + "allocated_bytes_peak_delta": 23655018496, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 93048604160, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 91996227584, + "device": 5, + "inactive_split_bytes_current": 6045862912, + "inactive_split_bytes_peak": 12270207488, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 97479819264, + "reserved_bytes_peak": 97479819264, + "segment_count_current": 383, + "segment_count_peak": 383, + "timestamp_utc": "2026-08-01T14:45:03.280976Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8459684864, + "inactive_split_bytes_current_delta": 2413821952, + "inactive_split_bytes_peak": 15290946560, + "inactive_split_bytes_peak_delta": 3020739072, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121043419136, + "reserved_bytes_current_delta": 23563599872, + "reserved_bytes_peak": 121043419136, + "reserved_bytes_peak_delta": 23563599872, + "segment_count_current": 440, + "segment_count_current_delta": 57, + "segment_count_peak": 440, + "segment_count_peak_delta": 57, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000001_20260801T144504559763Z.pickle", + "timestamp_utc": "2026-08-01T14:45:04.674747Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T144504559763Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T144504559763Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1b6bc1630461e3c5b84269eeff88dd604724d861 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000001_20260801T144504559763Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:316bfc31942dc0b19defdffaa7077d17d01725e9cef1c6d3cf82d5697d9f301c +size 5947486 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T144505924393Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T144505924393Z.json new file mode 100644 index 0000000000000000000000000000000000000000..9572ee3bdf7bf4a7cad8201803a5a5ddfb012b77 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T144505924393Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116712243200, + "active_bytes_peak_delta": 359699968, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655278592, + "device": 5, + "inactive_split_bytes_current": 8042351616, + "inactive_split_bytes_peak": 15290946560, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:05.923989Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115655278592, + "allocated_bytes_peak_delta": 4032512, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116352543232, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115651246080, + "device": 5, + "inactive_split_bytes_current": 8459684864, + "inactive_split_bytes_peak": 15290946560, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121043419136, + "reserved_bytes_peak": 121043419136, + "segment_count_current": 440, + "segment_count_peak": 440, + "timestamp_utc": "2026-08-01T14:45:04.688891Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8042351616, + "inactive_split_bytes_current_delta": -417333248, + "inactive_split_bytes_peak": 15290946560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121867599872, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 121867599872, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 441, + "segment_count_current_delta": 1, + "segment_count_peak": 441, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000002_20260801T144505924393Z.pickle", + "timestamp_utc": "2026-08-01T14:45:06.029790Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T144505924393Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T144505924393Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..7ffc12723d3870df2ac13754439294c0f8bfa1ba --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000002_20260801T144505924393Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ca33a415dc9309dd8809878b8ccd9931997485c5a733370924018ac66c243b4 +size 6101181 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T144507273672Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T144507273672Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6c696cc796739eeb3b5ad1f937d527b58ebd7325 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T144507273672Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116712243200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 9803959296, + "inactive_split_bytes_peak": 15290946560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:07.273262Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115669413888, + "allocated_bytes_peak_delta": 14135296, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115655278592, + "device": 5, + "inactive_split_bytes_current": 8042351616, + "inactive_split_bytes_peak": 15290946560, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:06.043420Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 9803959296, + "inactive_split_bytes_current_delta": 1761607680, + "inactive_split_bytes_peak": 15290946560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121867599872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121867599872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000003_20260801T144507273672Z.pickle", + "timestamp_utc": "2026-08-01T14:45:07.384300Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T144507273672Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T144507273672Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d6132846fa6f8865cef46bc7d4d02f3b26e3694f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000003_20260801T144507273672Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e836e29b939ad5a9d6f6e3457868523bc2d2a9d151dfcc1b45807413babd650f +size 6622833 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T144508911446Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T144508911446Z.json new file mode 100644 index 0000000000000000000000000000000000000000..3c4ef97248c8667a5b7f531c60b4209a622d8935 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T144508911446Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116712243200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 8042351616, + "inactive_split_bytes_peak": 15290946560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:08.911033Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115669413888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 9803959296, + "inactive_split_bytes_peak": 15290946560, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:07.397939Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8042351616, + "inactive_split_bytes_current_delta": -1761607680, + "inactive_split_bytes_peak": 15290946560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121867599872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121867599872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000004_20260801T144508911446Z.pickle", + "timestamp_utc": "2026-08-01T14:45:09.019724Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T144508911446Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T144508911446Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d465cdf181b1dd82c862bb6cb856e3ec03c4478e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000004_20260801T144508911446Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49dda8610e2171cab76ece0030358f2dd67358c64a10403593c491a797fbe197 +size 6622707 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T144510252579Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T144510252579Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a37a9ecb443a27705c90a881bf8a0a4220025016 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T144510252579Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116712243200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 7572589568, + "inactive_split_bytes_peak": 15290946560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:10.252157Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115669413888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 8042351616, + "inactive_split_bytes_peak": 15290946560, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:09.033518Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7572589568, + "inactive_split_bytes_current_delta": -469762048, + "inactive_split_bytes_peak": 15290946560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121867599872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121867599872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000005_20260801T144510252579Z.pickle", + "timestamp_utc": "2026-08-01T14:45:10.361436Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T144510252579Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T144510252579Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a6bf51b87f361fdda59cb9267d629dc71fbf32aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000005_20260801T144510252579Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:540b8b6e69ded81fe1526e3d87b21454a6c732b4a42c99b16d5b2e55d52a9520 +size 6622268 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T144511613556Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T144511613556Z.json new file mode 100644 index 0000000000000000000000000000000000000000..7d4bc66506a2989469c34db07c20757edcd6b162 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T144511613556Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116712243200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 8040254464, + "inactive_split_bytes_peak": 15290946560, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:11.613148Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115669413888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 7572589568, + "inactive_split_bytes_peak": 15290946560, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:10.375172Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8040254464, + "inactive_split_bytes_current_delta": 467664896, + "inactive_split_bytes_peak": 15290946560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121867599872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121867599872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000006_20260801T144511613556Z.pickle", + "timestamp_utc": "2026-08-01T14:45:11.717620Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T144511613556Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T144511613556Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..68f9d281b99e5d593706bfc24e48996926a64256 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000006_20260801T144511613556Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e67819bf429cb43285dc457f04fef72149807e2732871eef10ce009e56a089b4 +size 6622285 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T144513084117Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T144513084117Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f6115f22f13a8ed5ac8764086f1adc46461acc0a --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T144513084117Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116712243200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 8688274432, + "inactive_split_bytes_peak": 15290946560, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:13.083707Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115669413888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 8040254464, + "inactive_split_bytes_peak": 15290946560, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:11.730697Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 8688274432, + "inactive_split_bytes_current_delta": 648019968, + "inactive_split_bytes_peak": 15290946560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121867599872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121867599872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000007_20260801T144513084117Z.pickle", + "timestamp_utc": "2026-08-01T14:45:13.189075Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T144513084117Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T144513084117Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1b263b6afecac6a4a80d294940d0515045ef9483 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000007_20260801T144513084117Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9475c416586bf7f5575a78f1a2895069ec5e7a07adf6e54a2f55a2ed83ef8126 +size 6622206 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T144514411845Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T144514411845Z.json new file mode 100644 index 0000000000000000000000000000000000000000..f98d923d165331ad75d0a1abcdded36b65521377 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T144514411845Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116712243200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 7809567744, + "inactive_split_bytes_peak": 15290946560, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:14.411449Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115669413888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 8688274432, + "inactive_split_bytes_peak": 15290946560, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:13.202215Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 7809567744, + "inactive_split_bytes_current_delta": -878706688, + "inactive_split_bytes_peak": 15290946560, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121867599872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121867599872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000008_20260801T144514411845Z.pickle", + "timestamp_utc": "2026-08-01T14:45:14.513972Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T144514411845Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T144514411845Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fb4c77d262ce6e552152b029d97a87fc8cead4a5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000008_20260801T144514411845Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98154e1ae284545f5ad30c4559d8b2c910adced669cc4454feae5ab5af03f755 +size 6622122 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T144614834125Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T144614834125Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d2c66aabb45b6a760683341b1971b101fb1500cf --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T144614834125Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 116712243200, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 10389064704, + "inactive_split_bytes_peak": 15937954816, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:46:14.833660Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 115669413888, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116712243200, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115669413888, + "device": 5, + "inactive_split_bytes_current": 7809567744, + "inactive_split_bytes_peak": 15290946560, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 5, + "reserved_bytes_current": 121867599872, + "reserved_bytes_peak": 121867599872, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:14.527091Z", + "world_size": 8 + }, + "device": 5, + "device_delta": 0, + "inactive_split_bytes_current": 10389064704, + "inactive_split_bytes_current_delta": 2579496960, + "inactive_split_bytes_peak": 15937954816, + "inactive_split_bytes_peak_delta": 647008256, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 5, + "rank_delta": 0, + "reserved_bytes_current": 121867599872, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121867599872, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0005_iter000009_20260801T144614834125Z.pickle", + "timestamp_utc": "2026-08-01T14:46:15.038643Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T144614834125Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T144614834125Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..582a29f5b5e3935d4ebdf25c967398ba0b631652 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0005_iter000009_20260801T144614834125Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bb88b675bc3bcc680fe0b0e8cb613172e8d1c0689773dc9aa738843bcff8572 +size 10542337 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T144503143634Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T144503143634Z.json new file mode 100644 index 0000000000000000000000000000000000000000..6cceb6cf12c53bd81fabc8ca9e4f7d41e9cad1e0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T144503143634Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 37208595456, + "active_bytes_peak": 92349022208, + "active_bytes_peak_delta": 56882685952, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92349022208, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 91996700672, + "device": 6, + "inactive_split_bytes_current": 6517722112, + "inactive_split_bytes_peak": 12507785216, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 98436120576, + "reserved_bytes_peak": 98436120576, + "segment_count_current": 383, + "segment_count_peak": 383, + "timestamp_utc": "2026-08-01T14:45:03.143023Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 37208595456, + "allocated_bytes_peak": 91996700672, + "allocated_bytes_peak_delta": 56530364416, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 6, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:43:54.202359Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 6517722112, + "inactive_split_bytes_current_delta": 6384903168, + "inactive_split_bytes_peak": 12507785216, + "inactive_split_bytes_peak_delta": 12347399168, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 98436120576, + "reserved_bytes_current_delta": 59192115200, + "reserved_bytes_peak": 98436120576, + "reserved_bytes_peak_delta": 59192115200, + "segment_count_current": 383, + "segment_count_current_delta": 312, + "segment_count_peak": 383, + "segment_count_peak_delta": 312, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000000_20260801T144503143634Z.pickle", + "timestamp_utc": "2026-08-01T14:45:03.259894Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T144503143634Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T144503143634Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dc993a5367e51f05c98af6dc0bfd2b7cdb314b21 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000000_20260801T144503143634Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69b8f2209355c1b5112c953aaac1d6bd6a3fc4584bbd94d389d00b0f6e1967e3 +size 5945278 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T144504559804Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T144504559804Z.json new file mode 100644 index 0000000000000000000000000000000000000000..124ebdfe18d6508b2a8a6985e22312be2dd6dcfa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T144504559804Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116352483840, + "active_bytes_peak_delta": 24003461632, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116352483840, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115647959552, + "device": 6, + "inactive_split_bytes_current": 8633748480, + "inactive_split_bytes_peak": 14881094656, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:04.559393Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115647959552, + "allocated_bytes_peak_delta": 23651258880, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 92349022208, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 91996700672, + "device": 6, + "inactive_split_bytes_current": 6517722112, + "inactive_split_bytes_peak": 12507785216, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 98436120576, + "reserved_bytes_peak": 98436120576, + "segment_count_current": 383, + "segment_count_peak": 383, + "timestamp_utc": "2026-08-01T14:45:03.277432Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8633748480, + "inactive_split_bytes_current_delta": 2116026368, + "inactive_split_bytes_peak": 14881094656, + "inactive_split_bytes_peak_delta": 2373309440, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121630621696, + "reserved_bytes_current_delta": 23194501120, + "reserved_bytes_peak": 121630621696, + "reserved_bytes_peak_delta": 23194501120, + "segment_count_current": 441, + "segment_count_current_delta": 58, + "segment_count_peak": 441, + "segment_count_peak_delta": 58, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000001_20260801T144504559804Z.pickle", + "timestamp_utc": "2026-08-01T14:45:04.674023Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T144504559804Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T144504559804Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..cb54a33f92219a47ccc0066cdc55aab9c3e0b65f --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000001_20260801T144504559804Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8918859c0eee83459e9797e80e466bcbaedaca7de563c9e6c0de1eff06bda3ef +size 5965146 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T144505924458Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T144505924458Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67e483142ea7e2680cfbf6b99a4ad876be752d53 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T144505924458Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116362504192, + "active_bytes_peak_delta": 10020352, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 9984314368, + "inactive_split_bytes_peak": 14881094656, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:05.924048Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115657861120, + "allocated_bytes_peak_delta": 9901568, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116352483840, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115647959552, + "device": 6, + "inactive_split_bytes_current": 8633748480, + "inactive_split_bytes_peak": 14881094656, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:04.688021Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9984314368, + "inactive_split_bytes_current_delta": 1350565888, + "inactive_split_bytes_peak": 14881094656, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121630621696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121630621696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000002_20260801T144505924458Z.pickle", + "timestamp_utc": "2026-08-01T14:45:06.037222Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T144505924458Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T144505924458Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..09c1c570e3960da3993d595ad39a0ab08819442c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000002_20260801T144505924458Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e746f579dc3d105595e6a83dfeb6ffb4f151a71c7ec5d21b5aa458530637e02 +size 6623637 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T144507273682Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T144507273682Z.json new file mode 100644 index 0000000000000000000000000000000000000000..c0ba147c37a2ec52759e9895052bf9af8fd504b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T144507273682Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116362504192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 8629554176, + "inactive_split_bytes_peak": 15350007808, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:07.273281Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115657861120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 9984314368, + "inactive_split_bytes_peak": 14881094656, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:06.054233Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 8629554176, + "inactive_split_bytes_current_delta": -1354760192, + "inactive_split_bytes_peak": 15350007808, + "inactive_split_bytes_peak_delta": 468913152, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121630621696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121630621696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000003_20260801T144507273682Z.pickle", + "timestamp_utc": "2026-08-01T14:45:07.385034Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T144507273682Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T144507273682Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c9a0ab1893dd435e0fa50caf3cd7df226fffe4c6 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000003_20260801T144507273682Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58fb4166fefa52e88acff81eb6db43516d9596f84e6c585da5275f24b484015a +size 6640787 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T144508911463Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T144508911463Z.json new file mode 100644 index 0000000000000000000000000000000000000000..82af9bae4f2a2edd00bef12b47f0977462d1ab37 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T144508911463Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116362504192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 9569078272, + "inactive_split_bytes_peak": 15350007808, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:08.911053Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115657861120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 8629554176, + "inactive_split_bytes_peak": 15350007808, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:07.398723Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9569078272, + "inactive_split_bytes_current_delta": 939524096, + "inactive_split_bytes_peak": 15350007808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121630621696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121630621696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000004_20260801T144508911463Z.pickle", + "timestamp_utc": "2026-08-01T14:45:09.018744Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T144508911463Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T144508911463Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c58f371c014629d1334b56eeaf43ba284f8b46f5 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000004_20260801T144508911463Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3374ff8e2ba55cc75f2c82e71f2540270fea9b39389a2ea0f4e46297959ca9ae +size 6622553 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T144510252606Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T144510252606Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fa2ddf3177c7ccc897d633524f8aeaf55212b4b8 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T144510252606Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116362504192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 7692127232, + "inactive_split_bytes_peak": 15350007808, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:10.252190Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115657861120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 9569078272, + "inactive_split_bytes_peak": 15350007808, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:09.032387Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 7692127232, + "inactive_split_bytes_current_delta": -1876951040, + "inactive_split_bytes_peak": 15350007808, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121630621696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121630621696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000005_20260801T144510252606Z.pickle", + "timestamp_utc": "2026-08-01T14:45:10.358522Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T144510252606Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T144510252606Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..2e0569d6893c97c825d3921657eb92f3b58999b9 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000005_20260801T144510252606Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc9162c17c3bb0563f338bc3432f9d46d0660499fad601fb0e9692ba1cc129f5 +size 6622895 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T144511613571Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T144511613571Z.json new file mode 100644 index 0000000000000000000000000000000000000000..dbb4e37c8ce040fc6fc1000d3fc8af579c416d46 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T144511613571Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116362504192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 10160475136, + "inactive_split_bytes_peak": 15800653824, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:11.613159Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115657861120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 7692127232, + "inactive_split_bytes_peak": 15350007808, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:10.372018Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10160475136, + "inactive_split_bytes_current_delta": 2468347904, + "inactive_split_bytes_peak": 15800653824, + "inactive_split_bytes_peak_delta": 450646016, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121630621696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121630621696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000006_20260801T144511613571Z.pickle", + "timestamp_utc": "2026-08-01T14:45:11.716660Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T144511613571Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T144511613571Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..6251b15313842744237045a9fa4bb66acee32f1b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000006_20260801T144511613571Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:880d1a01c7ffaba3c7f926e72476cb050ebd4fec7479cf45463972da76edca89 +size 6640622 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T144513084147Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T144513084147Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b6eb85b4537d64b466985cab84c268be7e90476b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T144513084147Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116362504192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 9514552320, + "inactive_split_bytes_peak": 15800653824, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:13.083730Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115657861120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 10160475136, + "inactive_split_bytes_peak": 15800653824, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:11.729799Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 9514552320, + "inactive_split_bytes_current_delta": -645922816, + "inactive_split_bytes_peak": 15800653824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121630621696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121630621696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000007_20260801T144513084147Z.pickle", + "timestamp_utc": "2026-08-01T14:45:13.187439Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T144513084147Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T144513084147Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..dfd830db7344b2b29146307ec11b02b82351f9ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000007_20260801T144513084147Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4dc59f309eb4d44021d64a108927af84307c77c9bb925d58af7bf5b10a1524a +size 6622554 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T144514411872Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T144514411872Z.json new file mode 100644 index 0000000000000000000000000000000000000000..d0f078f1990102f98b42322cc57a829685851063 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T144514411872Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440050688, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116362504192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 10219195392, + "inactive_split_bytes_peak": 15800653824, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:14.411471Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440050688, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115657861120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 9514552320, + "inactive_split_bytes_peak": 15800653824, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:13.200499Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10219195392, + "inactive_split_bytes_current_delta": 704643072, + "inactive_split_bytes_peak": 15800653824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 121630621696, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 121630621696, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 441, + "segment_count_current_delta": 0, + "segment_count_peak": 441, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000008_20260801T144514411872Z.pickle", + "timestamp_utc": "2026-08-01T14:45:14.519993Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T144514411872Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T144514411872Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..c6692b6582323853bef1e91adf1c620e8d9798bd --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000008_20260801T144514411872Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8429e379d7c3edd2f52897b38e72e2ef68e8fe79f0b68d4a806d9c69370a57b5 +size 6622349 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T144614834220Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T144614834220Z.json new file mode 100644 index 0000000000000000000000000000000000000000..69212edcbe3ae155b86c6d4ceb97423d09f1c0ae --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T144614834220Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165761536, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 116362504192, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165761536, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 61165761536, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 10271624192, + "inactive_split_bytes_peak": 15800653824, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 6, + "reserved_bytes_current": 122454802432, + "reserved_bytes_peak": 122454802432, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:46:14.833660Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165761536, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 115657861120, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440050688, + "active_bytes_peak": 116362504192, + "allocated_bytes_current": 72440050688, + "allocated_bytes_peak": 115657861120, + "device": 6, + "inactive_split_bytes_current": 10219195392, + "inactive_split_bytes_peak": 15800653824, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 6, + "reserved_bytes_current": 121630621696, + "reserved_bytes_peak": 121630621696, + "segment_count_current": 441, + "segment_count_peak": 441, + "timestamp_utc": "2026-08-01T14:45:14.535782Z", + "world_size": 8 + }, + "device": 6, + "device_delta": 0, + "inactive_split_bytes_current": 10271624192, + "inactive_split_bytes_current_delta": 52428800, + "inactive_split_bytes_peak": 15800653824, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 6, + "rank_delta": 0, + "reserved_bytes_current": 122454802432, + "reserved_bytes_current_delta": 824180736, + "reserved_bytes_peak": 122454802432, + "reserved_bytes_peak_delta": 824180736, + "segment_count_current": 442, + "segment_count_current_delta": 1, + "segment_count_peak": 442, + "segment_count_peak_delta": 1, + "snapshot_pickle": "snapshot_iteration_rank0006_iter000009_20260801T144614834220Z.pickle", + "timestamp_utc": "2026-08-01T14:46:15.047121Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T144614834220Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T144614834220Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..de85113e53590907f942f0c07ad889d679e0221e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0006_iter000009_20260801T144614834220Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:259dc2b21ababa4aa759d9b96525c6cec7e232fb403d13aa1385436aea5babc5 +size 10019261 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T144503143531Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T144503143531Z.json new file mode 100644 index 0000000000000000000000000000000000000000..0e1c554dfbdc27fb96c10c29942092836702783d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T144503143531Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 37208595968, + "active_bytes_peak": 92693501440, + "active_bytes_peak_delta": 57227165184, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 92693501440, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 92004564992, + "device": 7, + "inactive_split_bytes_current": 6637259264, + "inactive_split_bytes_peak": 12566505472, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 98436120576, + "reserved_bytes_peak": 98436120576, + "segment_count_current": 383, + "segment_count_peak": 383, + "timestamp_utc": "2026-08-01T14:45:03.142951Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 37208595968, + "allocated_bytes_peak": 92004564992, + "allocated_bytes_peak_delta": 56538228736, + "before": { + "active_bytes_current": 35231455232, + "active_bytes_peak": 35466336256, + "allocated_bytes_current": 35231455232, + "allocated_bytes_peak": 35466336256, + "device": 7, + "inactive_split_bytes_current": 132818944, + "inactive_split_bytes_peak": 160386048, + "iteration": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 39244005376, + "reserved_bytes_peak": 39244005376, + "segment_count_current": 71, + "segment_count_peak": 71, + "timestamp_utc": "2026-08-01T14:43:54.204106Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 6637259264, + "inactive_split_bytes_current_delta": 6504440320, + "inactive_split_bytes_peak": 12566505472, + "inactive_split_bytes_peak_delta": 12406119424, + "iteration": 0, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 98436120576, + "reserved_bytes_current_delta": 59192115200, + "reserved_bytes_peak": 98436120576, + "reserved_bytes_peak_delta": 59192115200, + "segment_count_current": 383, + "segment_count_current_delta": 312, + "segment_count_peak": 383, + "segment_count_peak_delta": 312, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000000_20260801T144503143531Z.pickle", + "timestamp_utc": "2026-08-01T14:45:03.261947Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T144503143531Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T144503143531Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..1ae028bf531584e604ad1c0c67c5803f75dcb097 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000000_20260801T144503143531Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf8258b414327c201de619ffab20bd4bf0d865312483ae87babd6bce9faa863a +size 5946814 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T144504559832Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T144504559832Z.json new file mode 100644 index 0000000000000000000000000000000000000000..4c49699b6309158d6655943535d46018f3e32f18 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T144504559832Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72441099776, + "active_bytes_current_delta": 1048576, + "active_bytes_peak": 116004416000, + "active_bytes_peak_delta": 23310914560, + "after": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 116004416000, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 115652094464, + "device": 7, + "inactive_split_bytes_current": 7749798400, + "inactive_split_bytes_peak": 15649214976, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:04.559421Z", + "world_size": 8 + }, + "allocated_bytes_current": 72441099776, + "allocated_bytes_current_delta": 1048576, + "allocated_bytes_peak": 115652094464, + "allocated_bytes_peak_delta": 23647529472, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 92693501440, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 92004564992, + "device": 7, + "inactive_split_bytes_current": 6637259264, + "inactive_split_bytes_peak": 12566505472, + "iteration": 1, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 98436120576, + "reserved_bytes_peak": 98436120576, + "segment_count_current": 383, + "segment_count_peak": 383, + "timestamp_utc": "2026-08-01T14:45:03.281730Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 7749798400, + "inactive_split_bytes_current_delta": 1112539136, + "inactive_split_bytes_peak": 15649214976, + "inactive_split_bytes_peak_delta": 3082709504, + "iteration": 1, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 122335264768, + "reserved_bytes_current_delta": 23899144192, + "reserved_bytes_peak": 122335264768, + "reserved_bytes_peak_delta": 23899144192, + "segment_count_current": 442, + "segment_count_current_delta": 59, + "segment_count_peak": 442, + "segment_count_peak_delta": 59, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000001_20260801T144504559832Z.pickle", + "timestamp_utc": "2026-08-01T14:45:04.678337Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T144504559832Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T144504559832Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..24fd3c693eb1b0bc9dba772a89a7f3f8a57b0a0e --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000001_20260801T144504559832Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc3343c256b0fb193ddf30110c0c8a47fb04a2af9f203a6b81bee1dee7e7035a +size 5952224 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T144505924387Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T144505924387Z.json new file mode 100644 index 0000000000000000000000000000000000000000..fcf3bb3906d4f21b0d4c44b1951fdecaf42f836c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T144505924387Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": -1048576, + "active_bytes_peak": 116711418368, + "active_bytes_peak_delta": 707002368, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115654453760, + "device": 7, + "inactive_split_bytes_current": 8990263808, + "inactive_split_bytes_peak": 15649214976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:05.923990Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": -1048576, + "allocated_bytes_peak": 115654453760, + "allocated_bytes_peak_delta": 2359296, + "before": { + "active_bytes_current": 72441099776, + "active_bytes_peak": 116004416000, + "allocated_bytes_current": 72441099776, + "allocated_bytes_peak": 115652094464, + "device": 7, + "inactive_split_bytes_current": 7749798400, + "inactive_split_bytes_peak": 15649214976, + "iteration": 2, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:04.694025Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8990263808, + "inactive_split_bytes_current_delta": 1240465408, + "inactive_split_bytes_peak": 15649214976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 2, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 122335264768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122335264768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 442, + "segment_count_current_delta": 0, + "segment_count_peak": 442, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000002_20260801T144505924387Z.pickle", + "timestamp_utc": "2026-08-01T14:45:06.036709Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T144505924387Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T144505924387Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..34dd9c41f31a176b803a1b6f8470c2bf71e95c45 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000002_20260801T144505924387Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddcaa05d04a8686217f5c2669a278fa027d289fa3141539808780179dd38f6f0 +size 6642925 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T144507273692Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T144507273692Z.json new file mode 100644 index 0000000000000000000000000000000000000000..a3f81d60f71ee14f0d249d405422579fb9481da0 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T144507273692Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116711418368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115654453760, + "device": 7, + "inactive_split_bytes_current": 9218853376, + "inactive_split_bytes_peak": 15649214976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:07.273296Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115654453760, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115654453760, + "device": 7, + "inactive_split_bytes_current": 8990263808, + "inactive_split_bytes_peak": 15649214976, + "iteration": 3, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:06.054611Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9218853376, + "inactive_split_bytes_current_delta": 228589568, + "inactive_split_bytes_peak": 15649214976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 3, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 122335264768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122335264768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 442, + "segment_count_current_delta": 0, + "segment_count_peak": 442, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000003_20260801T144507273692Z.pickle", + "timestamp_utc": "2026-08-01T14:45:07.384177Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T144507273692Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T144507273692Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d1f51b541eeb872a730efdb0eccf87bcdc18ab53 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000003_20260801T144507273692Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fabc180afcc3d73381e7acfe7be3aca0f31343991fe6ff023309bf7184a8cc03 +size 6624647 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T144508911486Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T144508911486Z.json new file mode 100644 index 0000000000000000000000000000000000000000..67ceebe8a8fdb444a385f84d4cc4a0479456043d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T144508911486Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116711418368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115662356992, + "device": 7, + "inactive_split_bytes_current": 10158377472, + "inactive_split_bytes_peak": 15649214976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:08.911080Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115662356992, + "allocated_bytes_peak_delta": 7903232, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115654453760, + "device": 7, + "inactive_split_bytes_current": 9218853376, + "inactive_split_bytes_peak": 15649214976, + "iteration": 4, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:07.399565Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 10158377472, + "inactive_split_bytes_current_delta": 939524096, + "inactive_split_bytes_peak": 15649214976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 4, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 122335264768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122335264768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 442, + "segment_count_current_delta": 0, + "segment_count_peak": 442, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000004_20260801T144508911486Z.pickle", + "timestamp_utc": "2026-08-01T14:45:09.018298Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T144508911486Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T144508911486Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a9e24208de7a6ac85208e584c7342930870da374 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000004_20260801T144508911486Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77be694fa63420a4f7f3af78770806b6fac162d6000ffe7b9441849d59534d33 +size 6624668 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T144510252629Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T144510252629Z.json new file mode 100644 index 0000000000000000000000000000000000000000..86b949aad9a159cbcddbbf6d0065ab5d639723ec --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T144510252629Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116711418368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115662356992, + "device": 7, + "inactive_split_bytes_current": 9044789760, + "inactive_split_bytes_peak": 15649214976, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:10.252230Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115662356992, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115662356992, + "device": 7, + "inactive_split_bytes_current": 10158377472, + "inactive_split_bytes_peak": 15649214976, + "iteration": 5, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:09.033493Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9044789760, + "inactive_split_bytes_current_delta": -1113587712, + "inactive_split_bytes_peak": 15649214976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 5, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 122335264768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122335264768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 442, + "segment_count_current_delta": 0, + "segment_count_peak": 442, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000005_20260801T144510252629Z.pickle", + "timestamp_utc": "2026-08-01T14:45:10.359632Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T144510252629Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T144510252629Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..5049fdaf9d27aa952f1562c050b469dd07ae7b9c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000005_20260801T144510252629Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a9222dbd81c2c220ce321b06cb89ba5478f7c78bcb94074c0d87ebb191dc50d +size 6642340 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T144511613553Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T144511613553Z.json new file mode 100644 index 0000000000000000000000000000000000000000..8ac4be66fed5331dac34373fa1e6de6a688a8f14 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T144511613553Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116711418368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115662356992, + "device": 7, + "inactive_split_bytes_current": 9164327424, + "inactive_split_bytes_peak": 15649214976, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:11.613149Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115662356992, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115662356992, + "device": 7, + "inactive_split_bytes_current": 9044789760, + "inactive_split_bytes_peak": 15649214976, + "iteration": 6, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:10.375641Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 9164327424, + "inactive_split_bytes_current_delta": 119537664, + "inactive_split_bytes_peak": 15649214976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 6, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 122335264768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122335264768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 442, + "segment_count_current_delta": 0, + "segment_count_peak": 442, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000006_20260801T144511613553Z.pickle", + "timestamp_utc": "2026-08-01T14:45:11.719985Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T144511613553Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T144511613553Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..fee7d3e8b7b1abab461fbfd96bc2950a5b4b9eef --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000006_20260801T144511613553Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13829d0cae6b594abaddbe11b289d401b31789c7a553040831f8c09c0a678eaf +size 6624331 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T144513084193Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T144513084193Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5bbe42689dd349c61759dfa201b81f4e95f02ecc --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T144513084193Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116711418368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115662356992, + "device": 7, + "inactive_split_bytes_current": 8459684352, + "inactive_split_bytes_peak": 15649214976, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:13.083761Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115662356992, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115662356992, + "device": 7, + "inactive_split_bytes_current": 9164327424, + "inactive_split_bytes_peak": 15649214976, + "iteration": 7, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:11.735168Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8459684352, + "inactive_split_bytes_current_delta": -704643072, + "inactive_split_bytes_peak": 15649214976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 7, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 122335264768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122335264768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 442, + "segment_count_current_delta": 0, + "segment_count_peak": 442, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000007_20260801T144513084193Z.pickle", + "timestamp_utc": "2026-08-01T14:45:13.192985Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T144513084193Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T144513084193Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0708153ad73f89e6b3a3b42663d316d25b6a9af1 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000007_20260801T144513084193Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac35001e16129b0504ad184187c621faa4ba202cd4d3c89eff29f922569d086d +size 6624040 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T144514411893Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T144514411893Z.json new file mode 100644 index 0000000000000000000000000000000000000000..5b374f7cfb32ac9237e2f8bd68abc264fb239c9d --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T144514411893Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 72440051200, + "active_bytes_current_delta": 0, + "active_bytes_peak": 116711418368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115662356992, + "device": 7, + "inactive_split_bytes_current": 8751188480, + "inactive_split_bytes_peak": 15649214976, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:14.411491Z", + "world_size": 8 + }, + "allocated_bytes_current": 72440051200, + "allocated_bytes_current_delta": 0, + "allocated_bytes_peak": 115662356992, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115662356992, + "device": 7, + "inactive_split_bytes_current": 8459684352, + "inactive_split_bytes_peak": 15649214976, + "iteration": 8, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:13.208313Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8751188480, + "inactive_split_bytes_current_delta": 291504128, + "inactive_split_bytes_peak": 15649214976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 8, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 122335264768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122335264768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 442, + "segment_count_current_delta": 0, + "segment_count_peak": 442, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000008_20260801T144514411893Z.pickle", + "timestamp_utc": "2026-08-01T14:45:14.517096Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T144514411893Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T144514411893Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..a30fee6f5ef384ad2203423170153bb4d3419471 --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000008_20260801T144514411893Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a716dd3498607bc1bd6e1ec1833adb9929db54dd177b00a0470d2f48c23dcdcc +size 6624155 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T144614834234Z.json b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T144614834234Z.json new file mode 100644 index 0000000000000000000000000000000000000000..b80ec607abe667facb2c654ef85764fc98b8206c --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T144614834234Z.json @@ -0,0 +1,79 @@ +{ + "active_bytes_current": 61165762048, + "active_bytes_current_delta": -11274289152, + "active_bytes_peak": 116711418368, + "active_bytes_peak_delta": 0, + "after": { + "active_bytes_current": 61165762048, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 61165762048, + "allocated_bytes_peak": 115662356992, + "device": 7, + "inactive_split_bytes_current": 8744897024, + "inactive_split_bytes_peak": 15649214976, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_end", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:46:14.833667Z", + "world_size": 8 + }, + "allocated_bytes_current": 61165762048, + "allocated_bytes_current_delta": -11274289152, + "allocated_bytes_peak": 115662356992, + "allocated_bytes_peak_delta": 0, + "before": { + "active_bytes_current": 72440051200, + "active_bytes_peak": 116711418368, + "allocated_bytes_current": 72440051200, + "allocated_bytes_peak": 115662356992, + "device": 7, + "inactive_split_bytes_current": 8751188480, + "inactive_split_bytes_peak": 15649214976, + "iteration": 9, + "mode": "iteration", + "num_alloc_retries": 0, + "num_ooms": 0, + "phase": "iteration_start", + "rank": 7, + "reserved_bytes_current": 122335264768, + "reserved_bytes_peak": 122335264768, + "segment_count_current": 442, + "segment_count_peak": 442, + "timestamp_utc": "2026-08-01T14:45:14.532220Z", + "world_size": 8 + }, + "device": 7, + "device_delta": 0, + "inactive_split_bytes_current": 8744897024, + "inactive_split_bytes_current_delta": -6291456, + "inactive_split_bytes_peak": 15649214976, + "inactive_split_bytes_peak_delta": 0, + "iteration": 9, + "iteration_delta": 0, + "mode": "iteration", + "num_alloc_retries": 0, + "num_alloc_retries_delta": 0, + "num_ooms": 0, + "num_ooms_delta": 0, + "rank": 7, + "rank_delta": 0, + "reserved_bytes_current": 122335264768, + "reserved_bytes_current_delta": 0, + "reserved_bytes_peak": 122335264768, + "reserved_bytes_peak_delta": 0, + "segment_count_current": 442, + "segment_count_current_delta": 0, + "segment_count_peak": 442, + "segment_count_peak_delta": 0, + "snapshot_pickle": "snapshot_iteration_rank0007_iter000009_20260801T144614834234Z.pickle", + "timestamp_utc": "2026-08-01T14:46:15.046199Z", + "world_size": 8, + "world_size_delta": 0 +} \ No newline at end of file diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T144614834234Z.pickle b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T144614834234Z.pickle new file mode 100644 index 0000000000000000000000000000000000000000..0891ed40fdb21d0341ed41e751488ae86329a23b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on/snapshot_iteration_rank0007_iter000009_20260801T144614834234Z.pickle @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35b43389ad035d1ce77ac1b6327eaf479f71afa596069663335c500ccf822ec9 +size 10544038 diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/run_summary.tsv b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/run_summary.tsv new file mode 100644 index 0000000000000000000000000000000000000000..22df40ea0b59bde31aef84be9ba4ffd34051897b --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/run_summary.tsv @@ -0,0 +1,49 @@ +index cg seq_length capacity overlap recompute status pickles duration_s log_file +1 on 2048 1.00 off off SUCCESS 80 230 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off.log +2 on 2048 1.00 off on SUCCESS 80 240 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on.log +3 on 2048 1.00 on off SUCCESS 80 236 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off.log +4 on 2048 1.00 on on SUCCESS 80 231 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on.log +5 on 2048 1.25 off off SUCCESS 80 230 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off.log +6 on 2048 1.25 off on SUCCESS 80 231 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on.log +7 on 2048 1.25 on off SUCCESS 80 225 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off.log +8 on 2048 1.25 on on SUCCESS 80 230 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on.log +9 on 2048 1.50 off off SUCCESS 80 231 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off.log +10 on 2048 1.50 off on SUCCESS 80 236 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on.log +11 on 2048 1.50 on off SUCCESS 80 236 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off.log +12 on 2048 1.50 on on OOM 25 136 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on.log +13 on 4096 1.00 off off SUCCESS 80 241 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off.log +14 on 4096 1.00 off on SUCCESS 80 246 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on.log +15 on 4096 1.00 on off SUCCESS 80 241 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off.log +16 on 4096 1.00 on on OOM 32 137 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on.log +17 on 4096 1.25 off off SUCCESS 80 241 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off.log +18 on 4096 1.25 off on SUCCESS 80 240 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on.log +19 on 4096 1.25 on off OOM 32 133 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off.log +20 on 4096 1.25 on on OOM 32 139 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on.log +21 on 4096 1.50 off off SUCCESS 80 240 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off.log +22 on 4096 1.50 off on SUCCESS 80 241 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on.log +23 on 4096 1.50 on off PARTIAL 74 765 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off.log +24 on 4096 1.50 on on OOM 32 140 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/cg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on.log +25 off 2048 1.00 off off SUCCESS 80 226 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_off.log +26 off 2048 1.00 off on SUCCESS 80 226 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_off_recompute_on.log +27 off 2048 1.00 on off SUCCESS 80 226 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_off.log +28 off 2048 1.00 on on SUCCESS 80 225 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_00/dispatch_overlap_on_recompute_on.log +29 off 2048 1.25 off off SUCCESS 80 220 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_off.log +30 off 2048 1.25 off on SUCCESS 80 220 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_off_recompute_on.log +31 off 2048 1.25 on off SUCCESS 80 220 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_off.log +32 off 2048 1.25 on on SUCCESS 80 220 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_25/dispatch_overlap_on_recompute_on.log +33 off 2048 1.50 off off SUCCESS 80 226 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_off.log +34 off 2048 1.50 off on SUCCESS 80 226 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_off_recompute_on.log +35 off 2048 1.50 on off SUCCESS 80 220 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_off.log +36 off 2048 1.50 on on SUCCESS 80 225 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_2048/cap_1_50/dispatch_overlap_on_recompute_on.log +37 off 4096 1.00 off off SUCCESS 80 225 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_off.log +38 off 4096 1.00 off on SUCCESS 80 226 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_off_recompute_on.log +39 off 4096 1.00 on off SUCCESS 80 226 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_off.log +40 off 4096 1.00 on on SUCCESS 80 226 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_00/dispatch_overlap_on_recompute_on.log +41 off 4096 1.25 off off SUCCESS 80 226 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_off.log +42 off 4096 1.25 off on SUCCESS 80 231 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_off_recompute_on.log +43 off 4096 1.25 on off SUCCESS 80 221 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_off.log +44 off 4096 1.25 on on SUCCESS 80 230 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_25/dispatch_overlap_on_recompute_on.log +45 off 4096 1.50 off off SUCCESS 80 225 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_off.log +46 off 4096 1.50 off on SUCCESS 80 230 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_off_recompute_on.log +47 off 4096 1.50 on off SUCCESS 80 231 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_off.log +48 off 4096 1.50 on on SUCCESS 80 236 /workspace/blackwell_run_scripts/logs/run_mem_profile_dispatch_overlap_recompute_20260801_113835_439864/nocg/seq_4096/cap_1_50/dispatch_overlap_on_recompute_on.log diff --git a/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/sweep_config.txt b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/sweep_config.txt new file mode 100644 index 0000000000000000000000000000000000000000..91ecb1df1c6fe7249633faff3220f45a0c27f8aa --- /dev/null +++ b/moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864/sweep_config.txt @@ -0,0 +1,12 @@ +megatron_branch: better-memory-dump-support +global_batch_size: 32 +micro_batch_size: 1 +train_iters: 10 +cuda_graph_modes: on off +sequence_lengths: 2048 4096 +capacities: 1.00 1.25 1.50 +overlap_modes: off on +recompute_modes: off on +expected_runs: 48 +expected_pickles_per_run: 80 +pickle_root: /workspace/blackwell_run_scripts/../moe_echo_memory_bloat_dispatch_sweep_20260801_113835_439864